forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-19 13:21:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

trie: Derivesha with stacktrie (#21407)

Browse source
This commit is contained in:
Daniel Liu 2025-03-25 15:44:35 +08:00 committed by Daniel Liu
parent a4c7d7f458
commit 54f73e471f
10 changed files with 762 additions and 277 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/block_validator.go
View file

@ -68,7 +68,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
return fmt.Errorf("uncle root hash mismatch: have %x, want %x", hash, header.UncleHash)
}
if hash := types.DeriveSha(block.Transactions(), new(trie.Trie)); hash != header.TxHash {
if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
}
if !v.bc.HasBlockAndFullState(block.ParentHash(), block.NumberU64()-1) {
@ -96,7 +96,7 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat
return fmt.Errorf("invalid bloom (remote: %x local: %x)", header.Bloom, rbloom)
}
// Tre receipt Trie's root (R = (Tr [[H1, R1], ... [Hn, R1]]))
receiptSha := types.DeriveSha(receipts, new(trie.Trie))
receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
if receiptSha != header.ReceiptHash {
return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
}

18
core/types/derive_sha.go
View file

@ -23,7 +23,6 @@ import (
"github.com/XinFinOrg/XDPoSChain/rlp"
)
// DerivableList is the interface which can derive the hash.
type DerivableList interface {
Len() int
GetRlp(i int) []byte
@ -39,7 +38,22 @@ type Hasher interface {
func DeriveSha(list DerivableList, hasher Hasher) common.Hash {
hasher.Reset()
keybuf := new(bytes.Buffer)
for i := 0; i < list.Len(); i++ {
// StackTrie requires values to be inserted in increasing
// hash order, which is not the order that `list` provides
// hashes in. This insertion sequence ensures that the
// order is correct.
for i := 1; i < list.Len() && i <= 0x7f; i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
}
if list.Len() > 0 {
keybuf.Reset()
rlp.Encode(keybuf, uint(0))
hasher.Update(keybuf.Bytes(), list.GetRlp(0))
}
for i := 0x80; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))

4
eth/downloader/queue.go
View file

@ -773,7 +773,7 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLi
defer q.lock.Unlock()
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Transactions(txLists[index]), new(trie.Trie)) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
if types.DeriveSha(types.Transactions(txLists[index]), trie.NewStackTrie(nil)) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
return errInvalidBody
}
result.Transactions = txLists[index]
@ -791,7 +791,7 @@ func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int,
defer q.lock.Unlock()
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Receipts(receiptList[index]), new(trie.Trie)) != header.ReceiptHash {
if types.DeriveSha(types.Receipts(receiptList[index]), trie.NewStackTrie(nil)) != header.ReceiptHash {
return errInvalidReceipt
}
result.Receipts = receiptList[index]

4
internal/ethapi/trie_proof_test.go
View file

@ -49,7 +49,7 @@ func TestTransactionProof(t *testing.T) {
transactions := types.Transactions([]*types.Transaction{t1, t2, t3, t4})
tr := deriveTrie(transactions)
// for verifying the proof
root := types.DeriveSha(transactions, new(trie.Trie))
root := types.DeriveSha(transactions, trie.NewStackTrie(nil))
for i := 0; i < transactions.Len(); i++ {
var proof proofPairList
keybuf := new(bytes.Buffer)
@ -82,7 +82,7 @@ func TestReceiptProof(t *testing.T) {
receipts := types.Receipts([]*types.Receipt{r1, r2, r3, r4})
tr := deriveTrie(receipts)
// for verifying the proof
root := types.DeriveSha(receipts, new(trie.Trie))
root := types.DeriveSha(receipts, trie.NewStackTrie(nil))
for i := 0; i < receipts.Len(); i++ {
var proof proofPairList
keybuf := new(bytes.Buffer)

7
trie/database.go
View file

@ -98,6 +98,11 @@ type rawNode []byte
func (n rawNode) cache() (hashNode, bool) { panic("this should never end up in a live trie") }
func (n rawNode) fstring(ind string) string { panic("this should never end up in a live trie") }
func (n rawNode) EncodeRLP(w io.Writer) error {
_, err := w.Write([]byte(n))
return err
}
// rawFullNode represents only the useful data content of a full Node, with the
// caches and flags stripped out to minimize its data storage. This type honors
// the same RLP encoding as the original parent.
@ -198,7 +203,7 @@ func forGatherChildren(n node, onChild func(hash common.Hash)) {
}
case hashNode:
onChild(common.BytesToHash(n))
case valueNode, nil:
case valueNode, nil, rawNode:
default:
panic(fmt.Sprintf("unknown Node type: %T", n))
}

29
trie/encoding.go
View file

@ -51,6 +51,35 @@ func hexToCompact(hex []byte) []byte {
return buf
}
// hexToCompactInPlace places the compact key in input buffer, returning the length
// needed for the representation
func hexToCompactInPlace(hex []byte) int {
var (
hexLen = len(hex) // length of the hex input
firstByte = byte(0)
)
// Check if we have a terminator there
if hexLen > 0 && hex[hexLen-1] == 16 {
firstByte = 1 << 5
hexLen-- // last part was the terminator, ignore that
}
var (
binLen = hexLen/2 + 1
ni = 0 // index in hex
bi = 1 // index in bin (compact)
)
if hexLen&1 == 1 {
firstByte |= 1 << 4 // odd flag
firstByte |= hex[0] // first nibble is contained in the first byte
ni++
}
for ; ni < hexLen; bi, ni = bi+1, ni+2 {
hex[bi] = hex[ni]<<4 | hex[ni+1]
}
hex[0] = firstByte
return binLen
}
func compactToHex(compact []byte) []byte {
if len(compact) == 0 {
return compact

36
trie/encoding_test.go
View file

@ -18,6 +18,8 @@ package trie
import (
"bytes"
"encoding/hex"
"math/rand"
"testing"
)
@ -75,6 +77,40 @@ func TestHexKeybytes(t *testing.T) {
}
}
func TestHexToCompactInPlace(t *testing.T) {
for i, keyS := range []string{
"00",
"060a040c0f000a090b040803010801010900080d090a0a0d0903000b10",
"10",
} {
hexBytes, _ := hex.DecodeString(keyS)
exp := hexToCompact(hexBytes)
sz := hexToCompactInPlace(hexBytes)
got := hexBytes[:sz]
if !bytes.Equal(exp, got) {
t.Fatalf("test %d: encoding err\ninp %v\ngot %x\nexp %x\n", i, keyS, got, exp)
}
}
}
func TestHexToCompactInPlaceRandom(t *testing.T) {
for i := 0; i < 10000; i++ {
l := rand.Intn(128)
key := make([]byte, l)
rand.Read(key)
hexBytes := keybytesToHex(key)
hexOrig := []byte(string(hexBytes))
exp := hexToCompact(hexBytes)
sz := hexToCompactInPlace(hexBytes)
got := hexBytes[:sz]
if !bytes.Equal(exp, got) {
t.Fatalf("encoding err \ncpt %x\nhex %x\ngot %x\nexp %x\n",
key, hexOrig, got, exp)
}
}
}
func BenchmarkHexToCompact(b *testing.B) {
testBytes := []byte{0, 15, 1, 12, 11, 8, 16 /*term*/}
for i := 0; i < b.N; i++ {

405
trie/stacktrie.go Normal file
View file

@ -0,0 +1,405 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"fmt"
"sync"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/rlp"
)
var stPool = sync.Pool{
New: func() interface{} {
return NewStackTrie(nil)
},
}
func stackTrieFromPool(db ethdb.KeyValueStore) *StackTrie {
st := stPool.Get().(*StackTrie)
st.db = db
return st
}
func returnToPool(st *StackTrie) {
st.Reset()
stPool.Put(st)
}
// StackTrie is a trie implementation that expects keys to be inserted
// in order. Once it determines that a subtree will no longer be inserted
// into, it will hash it and free up the memory it uses.
type StackTrie struct {
nodeType uint8 // node type (as in branch, ext, leaf)
val []byte // value contained by this node if it's a leaf
key []byte // key chunk covered by this (full|ext) node
keyOffset int // offset of the key chunk inside a full key
children [16]*StackTrie // list of children (for fullnodes and exts)
db ethdb.KeyValueStore // Pointer to the commit db, can be nil
}
// NewStackTrie allocates and initializes an empty trie.
func NewStackTrie(db ethdb.KeyValueStore) *StackTrie {
return &StackTrie{
nodeType: emptyNode,
db: db,
}
}
func newLeaf(ko int, key, val []byte, db ethdb.KeyValueStore) *StackTrie {
st := stackTrieFromPool(db)
st.nodeType = leafNode
st.keyOffset = ko
st.key = append(st.key, key[ko:]...)
st.val = val
return st
}
func newExt(ko int, key []byte, child *StackTrie, db ethdb.KeyValueStore) *StackTrie {
st := stackTrieFromPool(db)
st.nodeType = extNode
st.keyOffset = ko
st.key = append(st.key, key[ko:]...)
st.children[0] = child
return st
}
// List all values that StackTrie#nodeType can hold
const (
emptyNode = iota
branchNode
extNode
leafNode
hashedNode
)
// TryUpdate inserts a (key, value) pair into the stack trie
func (st *StackTrie) TryUpdate(key, value []byte) error {
k := keybytesToHex(key)
if len(value) == 0 {
panic("deletion not supported")
}
st.insert(k[:len(k)-1], value)
return nil
}
func (st *StackTrie) Update(key, value []byte) {
if err := st.TryUpdate(key, value); err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
}
func (st *StackTrie) Reset() {
st.db = nil
st.key = st.key[:0]
st.val = st.val[:0]
for i := range st.children {
st.children[i] = nil
}
st.nodeType = emptyNode
st.keyOffset = 0
}
// Helper function that, given a full key, determines the index
// at which the chunk pointed by st.keyOffset is different from
// the same chunk in the full key.
func (st *StackTrie) getDiffIndex(key []byte) int {
diffindex := 0
for ; diffindex < len(st.key) && st.key[diffindex] == key[st.keyOffset+diffindex]; diffindex++ {
}
return diffindex
}
// Helper function to that inserts a (key, value) pair into
// the trie.
func (st *StackTrie) insert(key, value []byte) {
switch st.nodeType {
case branchNode: /* Branch */
idx := int(key[st.keyOffset])
// Unresolve elder siblings
for i := idx - 1; i >= 0; i-- {
if st.children[i] != nil {
if st.children[i].nodeType != hashedNode {
st.children[i].hash()
}
break
}
}
// Add new child
if st.children[idx] == nil {
st.children[idx] = stackTrieFromPool(st.db)
st.children[idx].keyOffset = st.keyOffset + 1
}
st.children[idx].insert(key, value)
case extNode: /* Ext */
// Compare both key chunks and see where they differ
diffidx := st.getDiffIndex(key)
// Check if chunks are identical. If so, recurse into
// the child node. Otherwise, the key has to be split
// into 1) an optional common prefix, 2) the fullnode
// representing the two differing path, and 3) a leaf
// for each of the differentiated subtrees.
if diffidx == len(st.key) {
// Ext key and key segment are identical, recurse into
// the child node.
st.children[0].insert(key, value)
return
}
// Save the original part. Depending if the break is
// at the extension's last byte or not, create an
// intermediate extension or use the extension's child
// node directly.
var n *StackTrie
if diffidx < len(st.key)-1 {
n = newExt(diffidx+1, st.key, st.children[0], st.db)
} else {
// Break on the last byte, no need to insert
// an extension node: reuse the current node
n = st.children[0]
}
// Convert to hash
n.hash()
var p *StackTrie
if diffidx == 0 {
// the break is on the first byte, so
// the current node is converted into
// a branch node.
st.children[0] = nil
p = st
st.nodeType = branchNode
} else {
// the common prefix is at least one byte
// long, insert a new intermediate branch
// node.
st.children[0] = stackTrieFromPool(st.db)
st.children[0].nodeType = branchNode
st.children[0].keyOffset = st.keyOffset + diffidx
p = st.children[0]
}
// Create a leaf for the inserted part
o := newLeaf(st.keyOffset+diffidx+1, key, value, st.db)
// Insert both child leaves where they belong:
origIdx := st.key[diffidx]
newIdx := key[diffidx+st.keyOffset]
p.children[origIdx] = n
p.children[newIdx] = o
st.key = st.key[:diffidx]
case leafNode: /* Leaf */
// Compare both key chunks and see where they differ
diffidx := st.getDiffIndex(key)
// Overwriting a key isn't supported, which means that
// the current leaf is expected to be split into 1) an
// optional extension for the common prefix of these 2
// keys, 2) a fullnode selecting the path on which the
// keys differ, and 3) one leaf for the differentiated
// component of each key.
if diffidx >= len(st.key) {
panic("Trying to insert into existing key")
}
// Check if the split occurs at the first nibble of the
// chunk. In that case, no prefix extnode is necessary.
// Otherwise, create that
var p *StackTrie
if diffidx == 0 {
// Convert current leaf into a branch
st.nodeType = branchNode
p = st
st.children[0] = nil
} else {
// Convert current node into an ext,
// and insert a child branch node.
st.nodeType = extNode
st.children[0] = NewStackTrie(st.db)
st.children[0].nodeType = branchNode
st.children[0].keyOffset = st.keyOffset + diffidx
p = st.children[0]
}
// Create the two child leaves: the one containing the
// original value and the one containing the new value
// The child leave will be hashed directly in order to
// free up some memory.
origIdx := st.key[diffidx]
p.children[origIdx] = newLeaf(diffidx+1, st.key, st.val, st.db)
p.children[origIdx].hash()
newIdx := key[diffidx+st.keyOffset]
p.children[newIdx] = newLeaf(p.keyOffset+1, key, value, st.db)
// Finally, cut off the key part that has been passed
// over to the children.
st.key = st.key[:diffidx]
st.val = nil
case emptyNode: /* Empty */
st.nodeType = leafNode
st.key = key[st.keyOffset:]
st.val = value
case hashedNode:
panic("trying to insert into hash")
default:
panic("invalid type")
}
}
// hash() hashes the node 'st' and converts it into 'hashedNode', if possible.
// Possible outcomes:
// 1. The rlp-encoded value was >= 32 bytes:
// - Then the 32-byte `hash` will be accessible in `st.val`.
// - And the 'st.type' will be 'hashedNode'
//
// 2. The rlp-encoded value was < 32 bytes
// - Then the <32 byte rlp-encoded value will be accessible in 'st.val'.
// - And the 'st.type' will be 'hashedNode' AGAIN
//
// This method will also:
// set 'st.type' to hashedNode
// clear 'st.key'
func (st *StackTrie) hash() {
/* Shortcut if node is already hashed */
if st.nodeType == hashedNode {
return
}
// The 'hasher' is taken from a pool, but we don't actually
// claim an instance until all children are done with their hashing,
// and we actually need one
var h *hasher
switch st.nodeType {
case branchNode:
var nodes [17]node
for i, child := range st.children {
if child == nil {
nodes[i] = nilValueNode
continue
}
child.hash()
if len(child.val) < 32 {
nodes[i] = rawNode(child.val)
} else {
nodes[i] = hashNode(child.val)
}
st.children[i] = nil // Reclaim mem from subtree
returnToPool(child)
}
nodes[16] = nilValueNode
h = newHasher(false)
defer returnHasherToPool(h)
h.tmp.Reset()
if err := rlp.Encode(&h.tmp, nodes); err != nil {
panic(err)
}
case extNode:
h = newHasher(false)
defer returnHasherToPool(h)
h.tmp.Reset()
st.children[0].hash()
// This is also possible:
//sz := hexToCompactInPlace(st.key)
//n := [][]byte{
// st.key[:sz],
// st.children[0].val,
//}
n := [][]byte{
hexToCompact(st.key),
st.children[0].val,
}
if err := rlp.Encode(&h.tmp, n); err != nil {
panic(err)
}
returnToPool(st.children[0])
st.children[0] = nil // Reclaim mem from subtree
case leafNode:
h = newHasher(false)
defer returnHasherToPool(h)
h.tmp.Reset()
st.key = append(st.key, byte(16))
sz := hexToCompactInPlace(st.key)
n := [][]byte{st.key[:sz], st.val}
if err := rlp.Encode(&h.tmp, n); err != nil {
panic(err)
}
case emptyNode:
st.val = st.val[:0]
st.val = append(st.val, emptyRoot[:]...)
st.key = st.key[:0]
st.nodeType = hashedNode
return
default:
panic("Invalid node type")
}
st.key = st.key[:0]
st.nodeType = hashedNode
if len(h.tmp) < 32 {
st.val = st.val[:0]
st.val = append(st.val, h.tmp...)
return
}
// Going to write the hash to the 'val'. Need to ensure it's properly sized first
// Typically, 'branchNode's will have no 'val', and require this allocation
if required := 32 - len(st.val); required > 0 {
buf := make([]byte, required)
st.val = append(st.val, buf...)
}
st.val = st.val[:32]
h.sha.Reset()
h.sha.Write(h.tmp)
h.sha.Read(st.val)
if st.db != nil {
// TODO! Is it safe to Put the slice here?
// Do all db implementations copy the value provided?
st.db.Put(st.val, h.tmp)
}
}
// Hash returns the hash of the current node
func (st *StackTrie) Hash() (h common.Hash) {
st.hash()
if len(st.val) != 32 {
// If the node's RLP isn't 32 bytes long, the node will not
// be hashed, and instead contain the rlp-encoding of the
// node. For the top level node, we need to force the hashing.
ret := make([]byte, 32)
h := newHasher(false)
defer returnHasherToPool(h)
h.sha.Reset()
h.sha.Write(st.val)
h.sha.Read(ret)
return common.BytesToHash(ret)
}
return common.BytesToHash(st.val)
}
// Commit will commit the current node to database db
func (st *StackTrie) Commit(db ethdb.KeyValueStore) common.Hash {
oldDb := st.db
st.db = db
defer func() {
st.db = oldDb
}()
st.hash()
h := common.BytesToHash(st.val)
return h
}

242
trie/stacktrie_test.go Normal file
View file

@ -0,0 +1,242 @@
package trie
import (
"bytes"
"fmt"
"math/big"
mrand "math/rand"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
)
func TestSizeBug(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
nt.TryUpdate(leaf, value)
st.TryUpdate(leaf, value)
if nt.Hash() != st.Hash() {
t.Fatalf("error %x != %x", st.Hash(), nt.Hash())
}
}
func TestEmptyBug(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
kvs := []struct {
K string
V string
}{
{K: "405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace", V: "9496f4ec2bf9dab484cac6be589e8417d84781be08"},
{K: "40edb63a35fcf86c08022722aa3287cdd36440d671b4918131b2514795fefa9c", V: "01"},
{K: "b10e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6", V: "947a30f7736e48d6599356464ba4c150d8da0302ff"},
{K: "c2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b", V: "02"},
}
for _, kv := range kvs {
nt.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
st.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
}
if nt.Hash() != st.Hash() {
t.Fatalf("error %x != %x", st.Hash(), nt.Hash())
}
}
func TestValLength56(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
//leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
//value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3")
kvs := []struct {
K string
V string
}{
{K: "405787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace", V: "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"},
}
for _, kv := range kvs {
nt.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
st.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
}
if nt.Hash() != st.Hash() {
t.Fatalf("error %x != %x", st.Hash(), nt.Hash())
}
}
func genTxs(num uint64) (types.Transactions, error) {
key, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
if err != nil {
return nil, err
}
var addr = crypto.PubkeyToAddress(key.PublicKey)
newTx := func(i uint64) (*types.Transaction, error) {
signer := types.NewEIP155Signer(big.NewInt(18))
tx, err := types.SignTx(types.NewTransaction(i, addr, new(big.Int), 0, new(big.Int).SetUint64(10000000), nil), signer, key)
return tx, err
}
var txs types.Transactions
for i := uint64(0); i < num; i++ {
tx, err := newTx(i)
if err != nil {
return nil, err
}
txs = append(txs, tx)
}
return txs, nil
}
func TestDeriveSha(t *testing.T) {
txs, err := genTxs(0)
if err != nil {
t.Fatal(err)
}
for len(txs) < 1000 {
exp := types.DeriveSha(txs, newEmpty())
got := types.DeriveSha(txs, NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
}
newTxs, err := genTxs(uint64(len(txs) + 1))
if err != nil {
t.Fatal(err)
}
txs = append(txs, newTxs...)
}
}
func BenchmarkDeriveSha200(b *testing.B) {
txs, err := genTxs(200)
if err != nil {
b.Fatal(err)
}
var exp common.Hash
var got common.Hash
b.Run("std_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, newEmpty())
}
})
b.Run("stack_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
got = types.DeriveSha(txs, NewStackTrie(nil))
}
})
if got != exp {
b.Errorf("got %x exp %x", got, exp)
}
}
type dummyDerivableList struct {
len int
seed int
}
func newDummy(seed int) *dummyDerivableList {
d := &dummyDerivableList{}
src := mrand.NewSource(int64(seed))
// don't use lists longer than 4K items
d.len = int(src.Int63() & 0x0FFF)
d.seed = seed
return d
}
func (d *dummyDerivableList) Len() int {
return d.len
}
func (d *dummyDerivableList) GetRlp(i int) []byte {
src := mrand.NewSource(int64(d.seed + i))
// max item size 256, at least 1 byte per item
size := 1 + src.Int63()&0x00FF
data := make([]byte, size)
_, err := mrand.New(src).Read(data)
if err != nil {
panic(err)
}
return data
}
func printList(l types.DerivableList) {
fmt.Printf("list length: %d\n", l.Len())
fmt.Printf("{\n")
for i := 0; i < l.Len(); i++ {
v := l.GetRlp(i)
fmt.Printf("\"0x%x\",\n", v)
}
fmt.Printf("},\n")
}
func TestFuzzDeriveSha(t *testing.T) {
// increase this for longer runs -- it's set to quite low for travis
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), newEmpty())
got := types.DeriveSha(newDummy(i), NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
t.Fatalf("seed %d: got %x exp %x", seed, got, exp)
}
}
}
type flatList struct {
rlpvals []string
}
func newFlatList(rlpvals []string) *flatList {
return &flatList{rlpvals}
}
func (f *flatList) Len() int {
return len(f.rlpvals)
}
func (f *flatList) GetRlp(i int) []byte {
return hexutil.MustDecode(f.rlpvals[i])
}
// TestDerivableList contains testcases found via fuzzing
func TestDerivableList(t *testing.T) {
type tcase []string
tcs := []tcase{
{
"0xc041",
},
{
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
},
{
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
"0x6cd850eca0a7ac46bb1748d7b9cb88aa3bd21c57d852c28198ad8fa422c4595032e88a4494b4778b36b944fe47a52b8c5cd312910139dfcb4147ab8e972cc456bcb063f25dd78f54c4d34679e03142c42c662af52947d45bdb6e555751334ace76a5080ab5a0256a1d259855dfc5c0b8023b25befbb13fd3684f9f755cbd3d63544c78ee2001452dd54633a7593ade0b183891a0a4e9c7844e1254005fbe592b1b89149a502c24b6e1dca44c158aebedf01beae9c30cabe16a",
"0x14abd5c47c0be87b0454596baad2",
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
},
}
for i, tc := range tcs[1:] {
exp := types.DeriveSha(newFlatList(tc), newEmpty())
got := types.DeriveSha(newFlatList(tc), NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)
}
}
}

290
trie/trie_test.go
View file

@ -19,19 +19,15 @@ package trie
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"math/big"
"math/rand"
"os"
"reflect"
"testing"
"testing/quick"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"
"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/davecgh/go-spew/spew"
@ -44,7 +40,7 @@ func init() {
// Used for testing
func newEmpty() *Trie {
trie, _ := New(emptyRoot, NewDatabase(memorydb.New()))
trie, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
return trie
}
@ -84,7 +80,7 @@ func testMissingNode(t *testing.T, memonly bool) {
diskdb := memorydb.New()
triedb := NewDatabase(diskdb)
trie, _ := New(emptyRoot, triedb)
trie, _ := New(common.Hash{}, triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _ := trie.Commit(nil)
@ -253,63 +249,6 @@ func TestEmptyValues(t *testing.T) {
}
}
func TestReplication(t *testing.T) {
trie := newEmpty()
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
{"horse", "stallion"},
{"shaman", "horse"},
{"doge", "coin"},
{"dog", "puppy"},
{"somethingveryoddindeedthis is", "myothernodedata"},
}
for _, val := range vals {
updateString(trie, val.k, val.v)
}
exp, err := trie.Commit(nil)
if err != nil {
t.Fatalf("commit error: %v", err)
}
// create a new trie on top of the database and check that lookups work.
trie2, err := New(exp, trie.Db)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
for _, kv := range vals {
if string(getString(trie2, kv.k)) != kv.v {
t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
}
}
hash, err := trie2.Commit(nil)
if err != nil {
t.Fatalf("commit error: %v", err)
}
if hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
}
// perform some insertions on the new trie.
vals2 := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
{"horse", "stallion"},
// {"shaman", "horse"},
// {"doge", "coin"},
// {"ether", ""},
// {"dog", "puppy"},
// {"somethingveryoddindeedthis is", "myothernodedata"},
// {"shaman", ""},
}
for _, val := range vals2 {
updateString(trie2, val.k, val.v)
}
if hash := trie2.Hash(); hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
}
}
func TestLargeValue(t *testing.T) {
trie := newEmpty()
trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
@ -347,9 +286,8 @@ func TestRandomCases(t *testing.T) {
{op: 1, key: common.Hex2Bytes("980c393656413a15c8da01978ed9f89feb80b502f58f2d640e3a2f5f7a99a7018f1b573befd92053ac6f78fca4a87268"), value: common.Hex2Bytes("")}, // step 24
{op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 25
}
if err := runRandTest(rt); err != nil {
t.Fatal(err)
}
runRandTest(rt)
}
// randTest performs random trie operations.
@ -375,56 +313,39 @@ const (
)
func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
var finishedFn = func() bool {
size--
return size > 0
}
return reflect.ValueOf(generateSteps(finishedFn, r))
}
func generateSteps(finished func() bool, r io.Reader) randTest {
var allKeys [][]byte
var one = []byte{0}
genKey := func() []byte {
r.Read(one)
if len(allKeys) < 2 || one[0]%100 > 90 {
if len(allKeys) < 2 || r.Intn(100) < 10 {
// new key
size := one[0] % 50
key := make([]byte, size)
key := make([]byte, r.Intn(50))
r.Read(key)
allKeys = append(allKeys, key)
return key
}
// use existing key
idx := int(one[0]) % len(allKeys)
return allKeys[idx]
return allKeys[r.Intn(len(allKeys))]
}
var steps randTest
for !finished() {
r.Read(one)
step := randTestStep{op: int(one[0]) % opMax}
for i := 0; i < size; i++ {
step := randTestStep{op: r.Intn(opMax)}
switch step.op {
case opUpdate:
step.key = genKey()
step.value = make([]byte, 8)
binary.BigEndian.PutUint64(step.value, uint64(len(steps)))
binary.BigEndian.PutUint64(step.value, uint64(i))
case opGet, opDelete:
step.key = genKey()
}
steps = append(steps, step)
}
return steps
return reflect.ValueOf(steps)
}
// runRandTestBool coerces error to boolean, for use in quick.Check
func runRandTestBool(rt randTest) bool {
return runRandTest(rt) == nil
}
func runRandTest(rt randTest) error {
func runRandTest(rt randTest) bool {
triedb := NewDatabase(memorydb.New())
tr, _ := New(emptyRoot, triedb)
tr, _ := New(common.Hash{}, triedb)
values := make(map[string]string) // tracks content of the trie
for i, step := range rt {
@ -441,7 +362,7 @@ func runRandTest(rt randTest) error {
v := tr.Get(step.key)
want := values[string(step.key)]
if string(v) != want {
rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want)
rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want)
}
case opCommit:
_, rt[i].err = tr.Commit(nil)
@ -451,34 +372,34 @@ func runRandTest(rt randTest) error {
hash, err := tr.Commit(nil)
if err != nil {
rt[i].err = err
return err
return false
}
newtr, err := New(hash, triedb)
if err != nil {
rt[i].err = err
return err
return false
}
tr = newtr
case opItercheckhash:
checktr, _ := New(emptyRoot, triedb)
checktr, _ := New(common.Hash{}, triedb)
it := NewIterator(tr.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
}
if tr.Hash() != checktr.Hash() {
rt[i].err = errors.New("hash mismatch in opItercheckhash")
rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash")
}
}
// Abort the test on error.
if rt[i].err != nil {
return rt[i].err
return false
}
}
return nil
return true
}
func TestRandom(t *testing.T) {
if err := quick.Check(runRandTestBool, nil); err != nil {
if err := quick.Check(runRandTest, nil); err != nil {
if cerr, ok := err.(*quick.CheckError); ok {
t.Fatalf("random test iteration %d failed: %s", cerr.Count, spew.Sdump(cerr.In))
}
@ -486,42 +407,9 @@ func TestRandom(t *testing.T) {
}
}
func BenchmarkGet(b *testing.B) { benchGet(b, false) }
func BenchmarkGetDB(b *testing.B) { benchGet(b, true) }
func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) }
func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000
func benchGet(b *testing.B, commit bool) {
trie := new(Trie)
if commit {
tmpdb := tempDB(b)
trie, _ = New(emptyRoot, tmpdb)
}
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
binary.LittleEndian.PutUint64(k, uint64(i))
trie.Update(k, k)
}
binary.LittleEndian.PutUint64(k, benchElemCount/2)
if commit {
trie.Commit(nil)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
trie.Get(k)
}
b.StopTimer()
if commit {
ldb := trie.Db.diskdb.(*leveldb.Database)
ldb.Close()
os.RemoveAll(ldb.Path())
}
}
func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
trie := newEmpty()
k := make([]byte, 32)
@ -575,65 +463,6 @@ type account struct {
Code []byte
}
// Benchmarks the trie Commit following a Hash. Since the trie caches the result of any operation,
// we cannot use b.N as the number of hashing rouns, since all rounds apart from
// the first one will be NOOP. As such, we'll use b.N as the number of account to
// insert into the trie before measuring the hashing.
func BenchmarkCommitAfterHash(b *testing.B) {
b.Run("no-onleaf", func(b *testing.B) {
benchmarkCommitAfterHash(b, nil)
})
var a account
onleaf := func(leaf []byte, parent common.Hash) error {
rlp.DecodeBytes(leaf, &a)
return nil
}
b.Run("with-onleaf", func(b *testing.B) {
benchmarkCommitAfterHash(b, onleaf)
})
}
func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) {
// Make the random benchmark deterministic
addresses, accounts := makeAccounts(b.N)
trie := newEmpty()
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
// Insert the accounts into the trie and hash it
trie.Hash()
b.ResetTimer()
b.ReportAllocs()
trie.Commit(onleaf)
}
func TestTinyTrie(t *testing.T) {
// Create a realistic account trie to hash
_, accounts := makeAccounts(10000)
trie := newEmpty()
trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3])
if exp, root := common.HexToHash("4fa6efd292cffa2db0083b8bedd23add2798ae73802442f52486e95c3df7111c"), trie.Hash(); exp != root {
t.Fatalf("1: got %x, exp %x", root, exp)
}
trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"), accounts[4])
if exp, root := common.HexToHash("cb5fb1213826dad9e604f095f8ceb5258fe6b5c01805ce6ef019a50699d2d479"), trie.Hash(); exp != root {
t.Fatalf("2: got %x, exp %x", root, exp)
}
trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"), accounts[4])
if exp, root := common.HexToHash("ed7e06b4010057d8703e7b9a160a6d42cf4021f9020da3c8891030349a646987"), trie.Hash(); exp != root {
t.Fatalf("3: got %x, exp %x", root, exp)
}
checktr, _ := New(emptyRoot, trie.Db)
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
}
if troot, itroot := trie.Hash(), checktr.Hash(); troot != itroot {
t.Fatalf("hash mismatch in opItercheckhash, trie: %x, check: %x", troot, itroot)
}
}
func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
@ -785,67 +614,6 @@ func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accou
b.StopTimer()
}
func BenchmarkDerefRootFixedSize(b *testing.B) {
b.Run("10", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(20)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("100", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(100)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("1K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(1000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("10K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(10000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("100K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(100000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
}
func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := newEmpty()
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
h := trie.Hash()
trie.Commit(nil)
b.StartTimer()
trie.Db.Dereference(h)
b.StopTimer()
}
func tempDB(tb testing.TB) *Database {
dir := tb.TempDir()
diskdb, err := leveldb.New(dir, 256, 0, "", false)
if err != nil {
panic(fmt.Sprintf("can't create temporary database: %v", err))
}
return NewDatabase(diskdb)
}
func getString(trie *Trie, k string) []byte {
return trie.Get([]byte(k))
}
@ -870,17 +638,3 @@ func TestDecodeNode(t *testing.T) {
decodeNode(hash, elems)
}
}
func FuzzTrie(f *testing.F) {
f.Fuzz(func(t *testing.T, data []byte) {
var steps = 500
var input = bytes.NewReader(data)
var finishedFn = func() bool {
steps--
return steps < 0 || input.Len() == 0
}
if err := runRandTest(generateSteps(finishedFn, input)); err != nil {
t.Fatal(err)
}
})
}