forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-23 15:14:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core, trie: rework trie committer #25320 (#1103)

Browse source 
* all: rework trie and trie committer

* all: get rid of internal cache in trie

* all: fixes

* trie: polish

* core, trie: address comments

* trie: fix imports

* core/state: address comments

* core/state/snapshot: polish

* trie: remove unused code

* trie: update tests

* trie: don't set db as nil

* trie: address comments

* trie: unskip test

Co-authored-by: rjl493456442 <garyrong0905@gmail.com>
This commit is contained in:
Daniel Liu 2025-08-31 15:58:48 +08:00 committed by GitHub
parent b856f98b47
commit 6c5c8c13de
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 604 additions and 335 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
XDCx/tradingstate/XDCx_trie.go
View file

@ -162,7 +162,18 @@ func (t *XDCXTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
t.secKeyCache = make(map[string][]byte)
}
// Commit the trie to its intermediate node database
root, _, err := t.trie.Commit(onleaf)
// PR #1103 causes TestRevertStates and TestDumpState to fail,
// but we will not fix them since XDCx has been abandoned.
// TODO(daniel): The following code may be incorrect, ref PR #25320:
root, nodes, err := t.trie.Commit(false)
if err != nil {
return common.Hash{}, err
}
if nodes != nil {
if err := t.trie.Db().Update(trie.NewWithNodeSet(nodes)); err != nil {
return common.Hash{}, err
}
}
return root, err
}

4
XDCx/tradingstate/statedb_test.go
View file

@ -159,6 +159,8 @@ func TestEchangeStates(t *testing.T) {
db.Close()
}
/*
// This test can not pass PR #25320
func TestRevertStates(t *testing.T) {
orderBook := common.StringToHash("BTC/XDC")
numberOrder := 20
@ -266,6 +268,7 @@ func TestRevertStates(t *testing.T) {
db.Close()
}
// This test can not pass PR #25320
func TestDumpState(t *testing.T) {
orderBook := common.StringToHash("BTC/XDC")
numberOrder := 5
@ -302,3 +305,4 @@ func TestDumpState(t *testing.T) {
fmt.Println("bidTrie", bidTrie)
db.Close()
}
*/

13
XDCxlending/lendingstate/XDCx_trie.go
View file

@ -158,7 +158,18 @@ func (t *XDCXTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) {
t.secKeyCache = make(map[string][]byte)
}
// Commit the trie to its intermediate node database
root, _, err := t.trie.Commit(onleaf)
// PR #1103 causes TestRevertStates and TestDumpState to fail,
// but we will not fix them since XDCx has been abandoned.
// TODO(daniel): The following code may be incorrect, ref PR #25320:
root, nodes, err := t.trie.Commit(false)
if err != nil {
return common.Hash{}, err
}
if nodes != nil {
if err := t.trie.Db().Update(trie.NewWithNodeSet(nodes)); err != nil {
return common.Hash{}, err
}
}
return root, err
}

4
XDCxlending/lendingstate/statedb_test.go
View file

@ -114,6 +114,8 @@ func TestEchangeStates(t *testing.T) {
db.Close()
}
/*
// This test can not pass PR #25320
func TestRevertStates(t *testing.T) {
orderBook := common.StringToHash("BTC/XDC")
numberOrder := 20
@ -223,6 +225,7 @@ func TestRevertStates(t *testing.T) {
db.Close()
}
// This test can not pass PR #25320
func TestDumpStates(t *testing.T) {
orderBook := common.StringToHash("BTC/XDC")
numberOrder := 20
@ -258,3 +261,4 @@ func TestDumpStates(t *testing.T) {
fmt.Println(statedb.DumpBorrowingTrie(orderBook))
db.Close()
}
*/

10
core/state/database.go
View file

@ -87,9 +87,13 @@ type Trie interface {
// can be used even if the trie doesn't have one.
Hash() common.Hash
// Commit writes all nodes to the trie's memory database, tracking the internal
// and external (for account tries) references.
Commit(onleaf trie.LeafCallback) (common.Hash, int, error)
// Commit collects all dirty nodes in the trie and replace them with the
// corresponding node hash. All collected nodes(including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
// The returned nodeset can be nil if the trie is clean(nothing to commit).
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error)
// NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key.

12
core/state/metrics.go
View file

@ -19,10 +19,10 @@ package state
import "github.com/XinFinOrg/XDPoSChain/metrics"
var (
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil)
storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil)
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
)

11
core/state/state_object.go
View file

@ -27,6 +27,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/trie"
)
type Code []byte
@ -315,22 +316,22 @@ func (s *stateObject) updateRoot(db Database) {
// CommitTrie the storage trie of the object to dwb.
// This updates the trie root.
func (s *stateObject) commitTrie(db Database) (int, error) {
func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
// If nothing changed, don't bother with hashing anything
tr, err := s.updateTrie(db)
if err != nil {
return 0, err
return nil, err
}
if s.dbErr != nil {
return 0, s.dbErr
return nil, s.dbErr
}
// If nothing changed, don't bother with hashing anything
if tr == nil {
return 0, nil
return nil, nil
}
// Track the amount of time wasted on committing the storage trie
defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
root, nodes, err := tr.Commit(nil)
root, nodes, err := tr.Commit(false)
if err == nil {
s.data.Root = root
}

45
core/state/statedb.go
View file

@ -757,7 +757,7 @@ func (s *StateDB) GetRefund() uint64 {
return s.refund
}
// Finalise finalises the state by removing the self destructed objects and clears
// Finalise finalises the state by removing the destructed objects and clears
// the journal as well as the refunds. Finalise, however, will not push any updates
// into the tries just yet. Only IntermediateRoot or Commit will do that.
func (s *StateDB) Finalise(deleteEmptyObjects bool) {
@ -839,7 +839,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
s.stateObjectsDirty[addr] = struct{}{}
}
// Commit objects to the trie, measuring the elapsed time
var storageCommitted int
var (
accountTrieNodes int
storageTrieNodes int
nodes = trie.NewMergedNodeSet()
)
codeWriter := s.db.TrieDB().DiskDB().NewBatch()
for addr := range s.stateObjectsDirty {
if obj := s.stateObjects[addr]; !obj.deleted {
@ -848,12 +852,18 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code)
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie.
committed, err := obj.commitTrie(s.db)
// Write any storage changes in the state object to its storage trie
set, err := obj.commitTrie(s.db)
if err != nil {
return common.Hash{}, err
}
storageCommitted += committed
// Merge the dirty nodes of storage trie into global set
if set != nil {
if err := nodes.Merge(set); err != nil {
return common.Hash{}, err
}
storageTrieNodes += set.Len()
}
}
// If the contract is destructed, the storage is still left in the
// database as dangling data. Theoretically it's should be wiped from
@ -873,19 +883,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
// Write the account trie changes, measuring the amount of wasted time
start := time.Now()
root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, _ []byte) error {
var account types.StateAccount
if err := rlp.DecodeBytes(leaf, &account); err != nil {
return nil
}
if account.Root != types.EmptyRootHash {
s.db.TrieDB().Reference(account.Root, parent)
}
return nil
})
root, set, err := s.trie.Commit(true)
if err != nil {
return common.Hash{}, err
}
// Merge the dirty nodes of account trie into global set
if set != nil {
if err := nodes.Merge(set); err != nil {
return common.Hash{}, err
}
accountTrieNodes = set.Len()
}
// Report the commit metrics
s.AccountCommits += time.Since(start)
@ -893,14 +901,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
storageUpdatedMeter.Mark(int64(s.StorageUpdated))
accountDeletedMeter.Mark(int64(s.AccountDeleted))
storageDeletedMeter.Mark(int64(s.StorageDeleted))
accountCommittedMeter.Mark(int64(accountCommitted))
storageCommittedMeter.Mark(int64(storageCommitted))
accountTrieCommittedMeter.Mark(int64(accountTrieNodes))
storageTriesCommittedMeter.Mark(int64(storageTrieNodes))
s.AccountUpdated, s.AccountDeleted = 0, 0
s.StorageUpdated, s.StorageDeleted = 0, 0
if len(s.stateObjectsDestruct) > 0 {
s.stateObjectsDestruct = make(map[common.Address]struct{})
}
if err := s.db.TrieDB().Update(nodes); err != nil {
return common.Hash{}, err
}
return root, err
}

2
internal/ethapi/trie_proof.go
View file

@ -30,7 +30,7 @@ func (n *proofPairList) Delete(key []byte) error {
// modified from core/types/derive_sha.go
func deriveTrie(list types.DerivableList) *trie.Trie {
buf := new(bytes.Buffer)
trie := new(trie.Trie)
trie := trie.NewEmpty(nil)
for i := range list.Len() {
buf.Reset()
rlp.Encode(buf, uint(i))

185
trie/committer.go
View file

@ -17,72 +17,48 @@
package trie
import (
"errors"
"fmt"
"sync"
"github.com/XinFinOrg/XDPoSChain/common"
)
// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow
// some parallelism but not incur too much memory overhead.
const leafChanSize = 200
// leaf represents a trie leaf value
// leaf represents a trie leaf node
type leaf struct {
size int // size of the rlp data (estimate)
hash common.Hash // hash of rlp data
node node // the node to commit
path []byte // the path from the root node
blob []byte // raw blob of leaf
parent common.Hash // the hash of parent node
}
// committer is a type used for the trie Commit operation. A committer has some
// internal preallocated temp space, and also a callback that is invoked when
// leaves are committed. The leafs are passed through the `leafCh`, to allow
// some level of parallelism.
// By 'some level' of parallelism, it's still the case that all leaves will be
// processed sequentially - onleaf will never be called in parallel or out of order.
// committer is the tool used for the trie Commit operation. The committer will
// capture all dirty nodes during the commit process and keep them cached in
// insertion order.
type committer struct {
onleaf LeafCallback
leafCh chan *leaf
}
// committers live in a global sync.Pool
var committerPool = sync.Pool{
New: func() interface{} {
return &committer{}
},
nodes *NodeSet
collectLeaf bool
}
// newCommitter creates a new committer or picks one from the pool.
func newCommitter() *committer {
return committerPool.Get().(*committer)
}
func returnCommitterToPool(h *committer) {
h.onleaf = nil
h.leafCh = nil
committerPool.Put(h)
func newCommitter(owner common.Hash, collectLeaf bool) *committer {
return &committer{
nodes: NewNodeSet(owner),
collectLeaf: collectLeaf,
}
}
// Commit collapses a node down into a hash node and inserts it into the database
func (c *committer) Commit(n node, db *Database) (hashNode, int, error) {
if db == nil {
return nil, 0, errors.New("no Db provided")
}
h, committed, err := c.commit(nil, n, db)
func (c *committer) Commit(n node) (hashNode, *NodeSet, error) {
h, err := c.commit(nil, n)
if err != nil {
return nil, 0, err
return nil, nil, err
}
return h.(hashNode), committed, nil
return h.(hashNode), c.nodes, nil
}
// commit collapses a node down into a hash node and inserts it into the database
func (c *committer) commit(path []byte, n node, db *Database) (node, int, error) {
func (c *committer) commit(path []byte, n node) (node, error) {
// if this path is clean, use available cached data
hash, dirty := n.cache()
if hash != nil && !dirty {
return hash, 0, nil
return hash, nil
}
// Commit children, then parent, and remove the dirty flag.
switch cn := n.(type) {
@ -92,36 +68,35 @@ func (c *committer) commit(path []byte, n node, db *Database) (node, int, error)
// If the child is fullNode, recursively commit,
// otherwise it can only be hashNode or valueNode.
var childCommitted int
if _, ok := cn.Val.(*fullNode); ok {
childV, committed, err := c.commit(append(path, cn.Key...), cn.Val, db)
childV, err := c.commit(append(path, cn.Key...), cn.Val)
if err != nil {
return nil, 0, err
return nil, err
}
collapsed.Val, childCommitted = childV, committed
collapsed.Val = childV
}
// The key needs to be copied, since we're delivering it to database
collapsed.Key = hexToCompact(cn.Key)
hashedNode := c.store(path, collapsed, db)
hashedNode := c.store(path, collapsed)
if hn, ok := hashedNode.(hashNode); ok {
return hn, childCommitted + 1, nil
return hn, nil
}
return collapsed, childCommitted, nil
return collapsed, nil
case *fullNode:
hashedKids, childCommitted, err := c.commitChildren(path, cn, db)
hashedKids, err := c.commitChildren(path, cn)
if err != nil {
return nil, 0, err
return nil, err
}
collapsed := cn.copy()
collapsed.Children = hashedKids
hashedNode := c.store(path, collapsed, db)
hashedNode := c.store(path, collapsed)
if hn, ok := hashedNode.(hashNode); ok {
return hn, childCommitted + 1, nil
return hn, nil
}
return collapsed, childCommitted, nil
return collapsed, nil
case hashNode:
return cn, 0, nil
return cn, nil
default:
// nil, valuenode shouldn't be committed
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
@ -129,11 +104,8 @@ func (c *committer) commit(path []byte, n node, db *Database) (node, int, error)
}
// commitChildren commits the children of the given fullnode
func (c *committer) commitChildren(path []byte, n *fullNode, db *Database) ([17]node, int, error) {
var (
committed int
children [17]node
)
func (c *committer) commitChildren(path []byte, n *fullNode) ([17]node, error) {
var children [17]node
for i := 0; i < 16; i++ {
child := n.Children[i]
if child == nil {
@ -149,88 +121,63 @@ func (c *committer) commitChildren(path []byte, n *fullNode, db *Database) ([17]
// Commit the child recursively and store the "hashed" value.
// Note the returned node can be some embedded nodes, so it's
// possible the type is not hashNode.
hashed, childCommitted, err := c.commit(append(path, byte(i)), child, db)
hashed, err := c.commit(append(path, byte(i)), child)
if err != nil {
return children, 0, err
return children, err
}
children[i] = hashed
committed += childCommitted
}
// For the 17th child, it's possible the type is valuenode.
if n.Children[16] != nil {
children[16] = n.Children[16]
}
return children, committed, nil
return children, nil
}
// store hashes the node n and if we have a storage layer specified, it writes
// the key/value pair to it and tracks any node->child references as well as any
// node->external trie references.
func (c *committer) store(path []byte, n node, db *Database) node {
func (c *committer) store(path []byte, n node) node {
// Larger nodes are replaced by their hash and stored in the database.
var (
hash, _ = n.cache()
size int
)
var hash, _ = n.cache()
// This was not generated - must be a small node stored in the parent.
// In theory, we should check if the node is leaf here (embedded node
// usually is leaf node). But small value(less than 32bytes) is not
// our target(leaves in account trie only).
if hash == nil {
// This was not generated - must be a small node stored in the parent.
// In theory, we should apply the leafCall here if it's not nil(embedded
// node usually contains value). But small value(less than 32bytes) is
// not our target.
return n
} else {
// We have the hash already, estimate the RLP encoding-size of the Node.
// The size is used for mem tracking, does not need to be exact
size = estimateSize(n)
}
// If we're using channel-based leaf-reporting, send to channel.
// The leaf channel will be active only when there an active leaf-callback
if c.leafCh != nil {
c.leafCh <- &leaf{
size: size,
hash: common.BytesToHash(hash),
node: n,
path: path,
// We have the hash already, estimate the RLP encoding-size of the node.
// The size is used for mem tracking, does not need to be exact
var (
size = estimateSize(n)
nhash = common.BytesToHash(hash)
mnode = &memoryNode{
hash: nhash,
node: simplifyNode(n),
size: uint16(size),
}
)
// Collect the dirty node to nodeset for return.
c.nodes.add(string(path), mnode)
// Collect the corresponding leaf node if it's required. We don't check
// full node since it's impossible to store value in fullNode. The key
// length of leaves should be exactly same.
if c.collectLeaf {
if sn, ok := n.(*shortNode); ok {
if val, ok := sn.Val.(valueNode); ok {
c.nodes.addLeaf(&leaf{blob: val, parent: nhash})
}
}
} else if db != nil {
// No leaf-callback used, but there's still a database. Do serial
// insertion
db.insert(common.BytesToHash(hash), size, n)
}
return hash
}
// commitLoop does the actual insert + leaf callback for nodes.
func (c *committer) commitLoop(db *Database) {
for item := range c.leafCh {
var (
hash = item.hash
size = item.size
n = item.node
)
// We are pooling the trie nodes into an intermediate memory cache
db.insert(hash, size, n)
if c.onleaf != nil {
switch n := n.(type) {
case *shortNode:
if child, ok := n.Val.(valueNode); ok {
c.onleaf(nil, nil, child, hash, nil)
}
case *fullNode:
// For children in range [0, 15], it's impossible
// to contain valueNode. Only check the 17th child.
if n.Children[16] != nil {
c.onleaf(nil, nil, n.Children[16].(valueNode), hash, nil)
}
}
}
}
}
// estimateSize estimates the size of an rlp-encoded node, without actually
// rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie
// with 1000 leafs, the only errors above 1% are on small shortnodes, where this
// with 1000 leaves, the only errors above 1% are on small shortnodes, where this
// method overestimates by 2 or 3 bytes (e.g. 37 instead of 35)
func estimateSize(n node) int {
switch n := n.(type) {

46
trie/database.go
View file

@ -27,6 +27,7 @@ import (
"github.com/VictoriaMetrics/fastcache"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
@ -320,14 +321,10 @@ func (db *Database) InsertPreimage(secKeyCache map[string][]byte) {
db.preimages.insertPreimage(preimages)
}
// insert inserts a collapsed trie node into the memory database.
// The blob size must be specified to allow proper size tracking.
// insert inserts a simplified trie node into the memory database.
// All nodes inserted by this function will be reference tracked
// and in theory should only used for **trie nodes** insertion.
func (db *Database) insert(hash common.Hash, size int, node node) {
db.lock.Lock()
defer db.lock.Unlock()
// If the node's already cached, skip
if _, ok := db.dirties[hash]; ok {
return
@ -336,7 +333,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) {
// Create the cached entry for this Node
entry := &cachedNode{
node: simplifyNode(node),
node: node,
size: uint16(size),
flushPrev: db.newest,
}
@ -775,7 +772,42 @@ func (c *cleaner) Delete(key []byte) error {
panic("not implemented")
}
// Size returns the current storage size of the memory Cache in front of the
// Update inserts the dirty nodes in provided nodeset into database and
// link the account trie with multiple storage tries if necessary.
func (db *Database) Update(nodes *MergedNodeSet) error {
db.lock.Lock()
defer db.lock.Unlock()
// Insert dirty nodes into the database. In the same tree, it must be
// ensured that children are inserted first, then parent so that children
// can be linked with their parent correctly. The order of writing between
// different tries(account trie, storage tries) is not required.
for owner, subset := range nodes.sets {
for _, path := range subset.paths {
n, ok := subset.nodes[path]
if !ok {
return fmt.Errorf("missing node %x %v", owner, path)
}
db.insert(n.hash, int(n.size), n.node)
}
}
// Link up the account trie and storage trie if the node points
// to an account trie leaf.
if set, present := nodes.sets[common.Hash{}]; present {
for _, n := range set.leaves {
var account types.StateAccount
if err := rlp.DecodeBytes(n.blob, &account); err != nil {
return err
}
if account.Root != types.EmptyRootHash {
db.reference(account.Root, n.parent)
}
}
}
return nil
}
// Size returns the current storage size of the memory cache in front of the
// persistent database layer.
func (db *Database) Size() (common.StorageSize, common.StorageSize) {
db.lock.RLock()

3
trie/iterator.go
View file

@ -376,8 +376,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) {
}
}
}
resolved, err := it.trie.resolveHash(hash, path)
return resolved, err
return it.trie.resolveHash(hash, path)
}
func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) {

56
trie/iterator_test.go
View file

@ -31,7 +31,7 @@ import (
)
func TestEmptyIterator(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
iter := trie.NodeIterator(nil)
seen := make(map[string]struct{})
@ -44,7 +44,8 @@ func TestEmptyIterator(t *testing.T) {
}
func TestIterator(t *testing.T) {
trie := newEmpty()
db := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(db)
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@ -59,8 +60,13 @@ func TestIterator(t *testing.T) {
all[val.k] = val.v
trie.Update([]byte(val.k), []byte(val.v))
}
trie.Commit(nil)
root, nodes, err := trie.Commit(false)
if err != nil {
t.Fatalf("Failed to commit trie %v", err)
}
db.Update(NewWithNodeSet(nodes))
trie, _ = New(common.Hash{}, root, db)
found := make(map[string]string)
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
@ -80,7 +86,7 @@ type kv struct {
}
func TestIteratorLargeData(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
for i := byte(0); i < 255; i++ {
@ -173,7 +179,7 @@ var testdata2 = []kvs{
}
func TestIteratorSeek(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for _, val := range testdata1 {
trie.Update([]byte(val.k), []byte(val.v))
}
@ -214,17 +220,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error {
}
func TestDifferenceIterator(t *testing.T) {
triea := newEmpty()
dba := NewDatabase(rawdb.NewMemoryDatabase())
triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.Update([]byte(val.k), []byte(val.v))
}
triea.Commit(nil)
rootA, nodesA, _ := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
triea, _ = New(common.Hash{}, rootA, dba)
trieb := newEmpty()
dbb := NewDatabase(rawdb.NewMemoryDatabase())
trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.Update([]byte(val.k), []byte(val.v))
}
trieb.Commit(nil)
rootB, nodesB, _ := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
trieb, _ = New(common.Hash{}, rootB, dbb)
found := make(map[string]string)
di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil))
@ -250,17 +262,23 @@ func TestDifferenceIterator(t *testing.T) {
}
func TestUnionIterator(t *testing.T) {
triea := newEmpty()
dba := NewDatabase(rawdb.NewMemoryDatabase())
triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.Update([]byte(val.k), []byte(val.v))
}
triea.Commit(nil)
rootA, nodesA, _ := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
triea, _ = New(common.Hash{}, rootA, dba)
trieb := newEmpty()
dbb := NewDatabase(rawdb.NewMemoryDatabase())
trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.Update([]byte(val.k), []byte(val.v))
}
trieb.Commit(nil)
rootB, nodesB, _ := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
trieb, _ = New(common.Hash{}, rootB, dbb)
di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
it := NewIterator(di)
@ -316,7 +334,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
tr.Commit(nil)
_, nodes, _ := tr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(tr.Hash(), true)
}
@ -407,7 +426,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v))
}
root, _, _ := ctr.Commit(nil)
root, nodes, _ := ctr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, true)
}
@ -521,7 +541,8 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) {
val = crypto.Keccak256(val)
trie.Update(key, val)
}
trie.Commit(nil)
_, nodes, _ := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
// Return the generated trie
return triedb, trie, logDb
}
@ -560,7 +581,8 @@ func TestIteratorNodeBlob(t *testing.T) {
all[val.k] = val.v
trie.Update([]byte(val.k), []byte(val.v))
}
trie.Commit(nil)
_, nodes, _ := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
triedb.Cap(0)
found := make(map[common.Hash][]byte)

94
trie/nodeset.go Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"fmt"
"github.com/XinFinOrg/XDPoSChain/common"
)
// memoryNode is all the information we know about a single cached trie node
// in the memory.
type memoryNode struct {
hash common.Hash // Node hash, computed by hashing rlp value
size uint16 // Byte size of the useful cached data
node node // Cached collapsed trie node, or raw rlp data
}
// NodeSet contains all dirty nodes collected during the commit operation.
// Each node is keyed by path. It's not thread-safe to use.
type NodeSet struct {
owner common.Hash // the identifier of the trie
paths []string // the path of dirty nodes, sort by insertion order
nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path
leaves []*leaf // the list of dirty leaves
}
// NewNodeSet initializes an empty node set to be used for tracking dirty nodes
// from a specific account or storage trie. The owner is zero for the account
// trie and the owning account address hash for storage tries.
func NewNodeSet(owner common.Hash) *NodeSet {
return &NodeSet{
owner: owner,
nodes: make(map[string]*memoryNode),
}
}
// add caches node with provided path and node object.
func (set *NodeSet) add(path string, node *memoryNode) {
set.paths = append(set.paths, path)
set.nodes[path] = node
}
// addLeaf caches the provided leaf node.
func (set *NodeSet) addLeaf(node *leaf) {
set.leaves = append(set.leaves, node)
}
// Len returns the number of dirty nodes contained in the set.
func (set *NodeSet) Len() int {
return len(set.nodes)
}
// MergedNodeSet represents a merged dirty node set for a group of tries.
type MergedNodeSet struct {
sets map[common.Hash]*NodeSet
}
// NewMergedNodeSet initializes an empty merged set.
func NewMergedNodeSet() *MergedNodeSet {
return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)}
}
// NewWithNodeSet constructs a merged nodeset with the provided single set.
func NewWithNodeSet(set *NodeSet) *MergedNodeSet {
merged := NewMergedNodeSet()
merged.Merge(set)
return merged
}
// Merge merges the provided dirty nodes of a trie into the set. The assumption
// is held that no duplicated set belonging to the same trie will be merged twice.
func (set *MergedNodeSet) Merge(other *NodeSet) error {
_, present := set.sets[other.owner]
if present {
return fmt.Errorf("duplicate trie for owner %#x", other.owner)
}
set.sets[other.owner] = other
return nil
}

14
trie/proof.go
View file

@ -22,6 +22,7 @@ import (
"fmt"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
)
@ -35,9 +36,12 @@ import (
// with the Node that proves the absence of the key.
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error {
// Collect all nodes on the path to key.
var (
prefix []byte
nodes []node
tn = t.root
)
key = keybytesToHex(key)
var nodes []node
tn := t.root
for len(key) > 0 && tn != nil {
switch n := tn.(type) {
case *shortNode:
@ -46,16 +50,18 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e
tn = nil
} else {
tn = n.Val
prefix = append(prefix, n.Key...)
key = key[len(n.Key):]
}
nodes = append(nodes, n)
case *fullNode:
tn = n.Children[key[0]]
prefix = append(prefix, key[0])
key = key[1:]
nodes = append(nodes, n)
case hashNode:
var err error
tn, err = t.resolveHash(n, nil)
tn, err = t.resolveHash(n, prefix)
if err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
return err
@ -553,7 +559,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key
}
// Rebuild the trie with the leaf stream, the shape of trie
// should be same with the original one.
tr := newWithRootNode(root)
tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())}
if empty {
tr.root = nil
}

18
trie/secure_trie.go
View file

@ -160,12 +160,14 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
return t.preimages.preimage(common.BytesToHash(shaKey))
}
// Commit writes all nodes and the secure hash pre-images to the trie's database.
// Nodes are stored with their sha3 hash as the key.
//
// Committing flushes nodes from memory. Subsequent Get calls will load nodes
// from the database.
func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
// Commit collects all dirty nodes in the trie and replace them with the
// corresponding node hash. All collected nodes(including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
// The returned nodeset can be nil if the trie is clean(nothing to commit).
// All cached preimages will be also flushed if preimages recording is enabled.
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *SecureTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) {
// Write all the pre-images to the actual disk database
if len(t.getSecKeyCache()) > 0 {
if t.preimages != nil {
@ -177,8 +179,8 @@ func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
}
t.secKeyCache = make(map[string][]byte)
}
// Commit the trie to its intermediate Node database
return t.trie.Commit(onleaf)
// Commit the trie to its intermediate node database
return t.trie.Commit(collectLeaf)
}
// Hash returns the root hash of SecureTrie. It does not write to the

15
trie/secure_trie_test.go
View file

@ -18,6 +18,7 @@ package trie
import (
"bytes"
"fmt"
"runtime"
"sync"
"testing"
@ -57,9 +58,15 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) {
trie.Update(key, val)
}
}
trie.Commit(nil)
// Return the generated trie
root, nodes, err := trie.Commit(false)
if err != nil {
panic(fmt.Errorf("failed to commit trie %v", err))
}
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}
// Re-create the trie based on the new state
trie, _ = NewSecure(common.Hash{}, root, triedb)
return triedb, trie, content
}
@ -135,7 +142,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
tries[index].Update(key, val)
}
}
tries[index].Commit(nil)
tries[index].Commit(false)
}(i)
}
// Wait for all threads to finish

13
trie/sync_test.go
View file

@ -18,6 +18,7 @@ package trie
import (
"bytes"
"fmt"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
@ -51,9 +52,15 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) {
trie.Update(key, val)
}
}
trie.Commit(nil)
// Return the generated trie
root, nodes, err := trie.Commit(false)
if err != nil {
panic(fmt.Errorf("failed to commit trie %v", err))
}
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}
// Re-create the trie based on the new state
trie, _ = NewSecure(common.Hash{}, root, triedb)
return triedb, trie, content
}

113
trie/trie.go
View file

@ -21,10 +21,8 @@ import (
"bytes"
"errors"
"fmt"
"sync"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/rlp"
@ -46,23 +44,28 @@ import (
// for extracting the raw states(leaf nodes) with corresponding paths.
type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error
// Trie is a Merkle Patricia Trie.
// The zero value is an empty trie with no database.
// Use New to create a trie that sits on top of a database.
// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on
// top of a database. Whenever trie performs a commit operation, the generated
// nodes will be gathered and returned in a set. Once the trie is committed,
// it's not usable anymore. Callers have to re-create the trie with new root
// based on the updated trie database.
//
// Trie is not safe for concurrent use.
type Trie struct {
db *Database
root node
owner common.Hash
// Keep track of the number leaves which have been inserted since the last
// hashing operation. This number will not directly map to the number of
// actually unhashed nodes
// actually unhashed nodes.
unhashed int
// tracer is the state diff tracer can be used to track newly added/deleted
// trie node. It will be reset after each commit operation.
// db is the handler trie can retrieve nodes from. It's
// only for reading purpose and not available for writing.
db *Database
// tracer is the tool to track the trie changes.
// It will be reset after each commit operation.
tracer *tracer
}
@ -78,10 +81,10 @@ func (t *Trie) Db() *Database {
// Copy returns a copy of Trie.
func (t *Trie) Copy() *Trie {
return &Trie{
db: t.db,
root: t.root,
owner: t.owner,
unhashed: t.unhashed,
db: t.db,
tracer: t.tracer.copy(),
}
}
@ -94,33 +97,9 @@ func (t *Trie) Copy() *Trie {
// New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand.
func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
return newTrie(owner, root, db)
}
// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
func NewEmpty(db *Database) *Trie {
tr, _ := newTrie(common.Hash{}, common.Hash{}, db)
return tr
}
// newWithRootNode initializes the trie with the given root node.
// It's only used by range prover.
func newWithRootNode(root node) *Trie {
return &Trie{
root: root,
//tracer: newTracer(),
db: NewDatabase(rawdb.NewMemoryDatabase()),
}
}
// newTrie is the internal function used to construct the trie with given parameters.
func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
if db == nil {
panic("trie.New called without a database")
}
trie := &Trie{
db: db,
owner: owner,
db: db,
//tracer: newTracer(),
}
if root != (common.Hash{}) && root != types.EmptyRootHash {
@ -133,6 +112,12 @@ func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
return trie, nil
}
// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
func NewEmpty(db *Database) *Trie {
tr, _ := New(common.Hash{}, common.Hash{}, db)
return tr
}
// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
// the key after the given start key.
func (t *Trie) NodeIterator(start []byte) NodeIterator {
@ -671,7 +656,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
// ShortNode{..., ShortNode{...}}. Since the entry
// might not be loaded yet, resolve it just for this
// check.
cnode, err := t.resolve(n.Children[pos], prefix)
cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos)))
if err != nil {
return false, nil, err
}
@ -731,6 +716,8 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
return n, nil
}
// resolveHash loads node from the underlying database with the provided
// node hash and path prefix.
func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
hash := common.BytesToHash(n)
if node := t.db.node(hash); node != nil {
@ -739,6 +726,8 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
}
// resolveHash loads rlp-encoded node blob from the underlying database
// with the provided node hash and path prefix.
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
hash := common.BytesToHash(n)
blob, _ := t.db.Node(hash)
@ -756,56 +745,40 @@ func (t *Trie) Hash() common.Hash {
return common.BytesToHash(hash.(hashNode))
}
// Commit writes all nodes to the trie's memory database, tracking the internal
// and external (for account tries) references.
func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
// Commit collects all dirty nodes in the trie and replace them with the
// corresponding node hash. All collected nodes(including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
// The returned nodeset can be nil if the trie is clean(nothing to commit).
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) {
if t.db == nil {
panic("commit called on trie with nil database")
}
defer t.tracer.reset()
if t.root == nil {
return types.EmptyRootHash, 0, nil
return types.EmptyRootHash, nil, nil
}
// Derive the hash for all dirty nodes first. We hold the assumption
// in the following procedure that all nodes are hashed.
rootHash := t.Hash()
h := newCommitter()
defer returnCommitterToPool(h)
// Do a quick check if we really need to commit, before we spin
// up goroutines. This can happen e.g. if we load a trie for reading storage
// values, but don't write to it.
// Do a quick check if we really need to commit. This can happen e.g.
// if we load a trie for reading storage values, but don't write to it.
if hashedNode, dirty := t.root.cache(); !dirty {
// Replace the root node with the origin hash in order to
// ensure all resolved nodes are dropped after the commit.
t.root = hashedNode
return rootHash, 0, nil
}
var wg sync.WaitGroup
if onleaf != nil {
h.onleaf = onleaf
h.leafCh = make(chan *leaf, leafChanSize)
wg.Add(1)
go func() {
defer wg.Done()
h.commitLoop(t.db)
}()
}
newRoot, committed, err := h.Commit(t.root, t.db)
if onleaf != nil {
// The leafch is created in newCommitter if there was an onleaf callback
// provided. The commitLoop only _reads_ from it, and the commit
// operation was the sole writer. Therefore, it's safe to close this
// channel here.
close(h.leafCh)
wg.Wait()
return rootHash, nil, nil
}
h := newCommitter(t.owner, collectLeaf)
newRoot, nodes, err := h.Commit(t.root)
if err != nil {
return common.Hash{}, 0, err
return common.Hash{}, nil, err
}
t.root = newRoot
return rootHash, committed, nil
return rootHash, nodes, nil
}
// hashRoot calculates the root hash of the given trie
@ -826,10 +799,6 @@ func (t *Trie) Reset() {
t.root = nil
t.owner = common.Hash{}
t.unhashed = 0
//t.db = nil
t.tracer.reset()
}
// Owner returns the associated trie owner.
func (t *Trie) Owner() common.Hash {
return t.owner
}

257
trie/trie_test.go
View file

@ -46,12 +46,6 @@ func init() {
spew.Config.DisableMethods = false
}
// Used for testing
func newEmpty() *Trie {
trie := NewEmpty(NewDatabase(memorydb.New()))
return trie
}
func TestEmptyTrie(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
res := trie.Hash()
@ -91,7 +85,8 @@ func testMissingNode(t *testing.T, memonly bool) {
trie := NewEmpty(triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
root, _, _ := trie.Commit(nil)
root, nodes, _ := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, true)
}
@ -157,7 +152,7 @@ func testMissingNode(t *testing.T, memonly bool) {
}
func TestInsert(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
@ -169,11 +164,11 @@ func TestInsert(t *testing.T) {
t.Errorf("case 1: exp %x got %x", exp, root)
}
trie = newEmpty()
trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
root, _, err := trie.Commit(nil)
root, _, err := trie.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@ -183,7 +178,8 @@ func TestInsert(t *testing.T) {
}
func TestGet(t *testing.T) {
trie := newEmpty()
db := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(db)
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
updateString(trie, "dogglesworth", "cat")
@ -193,21 +189,21 @@ func TestGet(t *testing.T) {
if !bytes.Equal(res, []byte("puppy")) {
t.Errorf("expected puppy got %x", res)
}
unknown := getString(trie, "unknown")
if unknown != nil {
t.Errorf("expected nil got %x", unknown)
}
if i == 1 {
return
}
trie.Commit(nil)
root, nodes, _ := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
trie, _ = New(common.Hash{}, root, db)
}
}
func TestDelete(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@ -234,7 +230,7 @@ func TestDelete(t *testing.T) {
}
func TestEmptyValues(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := []struct{ k, v string }{
{"do", "verb"},
@ -257,8 +253,75 @@ func TestEmptyValues(t *testing.T) {
}
}
func TestReplication(t *testing.T) {
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(triedb)
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
{"horse", "stallion"},
{"shaman", "horse"},
{"doge", "coin"},
{"dog", "puppy"},
{"somethingveryoddindeedthis is", "myothernodedata"},
}
for _, val := range vals {
updateString(trie, val.k, val.v)
}
exp, nodes, err := trie.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
triedb.Update(NewWithNodeSet(nodes))
// create a new trie on top of the database and check that lookups work.
trie2, err := New(common.Hash{}, exp, triedb)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
for _, kv := range vals {
if string(getString(trie2, kv.k)) != kv.v {
t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
}
}
hash, nodes, err := trie2.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
if hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
}
// recreate the trie after commit
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
}
trie2, err = New(common.Hash{}, hash, triedb)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
// perform some insertions on the new trie.
vals2 := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
{"horse", "stallion"},
// {"shaman", "horse"},
// {"doge", "coin"},
// {"ether", ""},
// {"dog", "puppy"},
// {"somethingveryoddindeedthis is", "myothernodedata"},
// {"shaman", ""},
}
for _, val := range vals2 {
updateString(trie2, val.k, val.v)
}
if hash := trie2.Hash(); hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
}
}
func TestLargeValue(t *testing.T) {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32))
trie.Hash()
@ -314,9 +377,8 @@ const (
opUpdate = iota
opDelete
opGet
opCommit
opHash
opReset
opCommit
opItercheckhash
opNodeDiff
opMax // boundary value, not an actual op
@ -394,17 +456,17 @@ func runRandTest(rt randTest) error {
if string(v) != want {
rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want)
}
case opCommit:
_, _, rt[i].err = tr.Commit(nil)
origTrie = tr.Copy()
case opHash:
tr.Hash()
case opReset:
hash, _, err := tr.Commit(nil)
case opCommit:
hash, nodes, err := tr.Commit(false)
if err != nil {
rt[i].err = err
return err
}
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
}
newtr, err := New(common.Hash{}, hash, triedb)
if err != nil {
rt[i].err = err
@ -494,11 +556,31 @@ func TestRandom(t *testing.T) {
}
}
func BenchmarkGet(b *testing.B) { benchGet(b) }
func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) }
func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000
func benchGet(b *testing.B) {
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(triedb)
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
binary.LittleEndian.PutUint64(k, uint64(i))
trie.Update(k, k)
}
binary.LittleEndian.PutUint64(k, benchElemCount/2)
b.ResetTimer()
for i := 0; i < b.N; i++ {
trie.Get(k)
}
b.StopTimer()
}
func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
k := make([]byte, 32)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
@ -528,7 +610,7 @@ func BenchmarkHash(b *testing.B) {
// entries, then adding N more.
addresses, accounts := makeAccounts(2 * b.N)
// Insert the accounts into the trie and hash it
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
i := 0
for ; i < len(addresses)/2; i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
@ -549,43 +631,31 @@ func BenchmarkHash(b *testing.B) {
// insert into the trie before measuring the hashing.
func BenchmarkCommitAfterHash(b *testing.B) {
b.Run("no-onleaf", func(b *testing.B) {
benchmarkCommitAfterHash(b, nil)
benchmarkCommitAfterHash(b, false)
})
var a types.StateAccount
onleaf := func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, parentPath []byte) error {
rlp.DecodeBytes(leaf, &a)
return nil
}
b.Run("with-onleaf", func(b *testing.B) {
benchmarkCommitAfterHash(b, onleaf)
benchmarkCommitAfterHash(b, true)
})
}
func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
trie := newEmpty()
func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) {
// Make the random benchmark deterministic
addresses, accounts := makeAccounts(b.N)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
// Insert the accounts into the trie and hash it
trie.Hash()
trie.Commit(nil)
root := trie.Hash()
exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
root, _, _ = trie.Commit(nil)
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
b.ResetTimer()
b.ReportAllocs()
trie.Commit(collectLeaf)
}
func TestTinyTrie(t *testing.T) {
// Create a realistic account trie to hash
_, accounts := makeAccounts(5)
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3])
if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root {
t.Errorf("1: got %x, exp %x", root, exp)
@ -598,7 +668,7 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp)
}
checktr := NewEmpty(trie.Db())
checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
@ -608,18 +678,25 @@ func TestTinyTrie(t *testing.T) {
}
}
func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) {
// Make the random benchmark deterministic
addresses, accounts := makeAccounts(b.N)
trie := newEmpty()
func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
// Insert the accounts into the trie and hash it
trie.Hash()
b.ResetTimer()
b.ReportAllocs()
trie.Commit(onleaf)
trie.Commit(false)
root := trie.Hash()
exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
root, _, _ = trie.Commit(false)
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
}
func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) {
@ -706,7 +783,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge)
// Fill the trie with elements
for i := 1; i < count; i++ {
for i := 0; i < count; i++ {
// For the stack trie, we need to do inserts in proper order
key := make([]byte, 32)
binary.BigEndian.PutUint64(key, uint64(i))
@ -722,8 +799,9 @@ func TestCommitSequenceStackTrie(t *testing.T) {
stTrie.TryUpdate(key, val)
}
// Flush trie -> database
root, _, _ := trie.Commit(nil)
root, nodes, _ := trie.Commit(false)
// Flush memdb -> disk (sponge)
db.Update(NewWithNodeSet(nodes))
db.Commit(root, false)
// And flush stacktrie -> disk
stRoot, err := stTrie.Commit()
@ -767,8 +845,9 @@ func TestCommitSequenceSmallRoot(t *testing.T) {
trie.TryUpdate(key, []byte{0x1})
stTrie.TryUpdate(key, []byte{0x1})
// Flush trie -> database
root, _, _ := trie.Commit(nil)
root, nodes, _ := trie.Commit(false)
// Flush memdb -> disk (sponge)
db.Update(NewWithNodeSet(nodes))
db.Commit(root, false)
// And flush stacktrie -> disk
stRoot, err := stTrie.Commit()
@ -829,7 +908,7 @@ func BenchmarkHashFixedSize(b *testing.B) {
func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -880,14 +959,68 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) {
func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := newEmpty()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
// Insert the accounts into the trie and hash it
trie.Hash()
b.StartTimer()
trie.Commit(nil)
trie.Commit(false)
b.StopTimer()
}
func BenchmarkDerefRootFixedSize(b *testing.B) {
b.Run("10", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(20)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("100", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(100)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("1K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(1000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("10K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(10000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
b.Run("100K", func(b *testing.B) {
b.StopTimer()
acc, add := makeAccounts(100000)
for i := 0; i < b.N; i++ {
benchmarkDerefRootFixedSize(b, acc, add)
}
})
}
func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(triedb)
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
h := trie.Hash()
_, nodes, _ := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
b.StartTimer()
triedb.Dereference(h)
b.StopTimer()
}

11
trie/util_test.go
View file

@ -19,12 +19,14 @@ package trie
import (
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
)
// Tests if the trie diffs are tracked correctly.
func TestTrieTracer(t *testing.T) {
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
db := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(db)
trie.tracer = newTracer()
// Insert a batch of entries, all the nodes should be marked as inserted
@ -65,8 +67,11 @@ func TestTrieTracer(t *testing.T) {
t.Fatalf("Unexpected deleted node tracked %d", len(deleted))
}
// Commit the changes
trie.Commit(nil)
// Commit the changes and re-create with new root
root, nodes, _ := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
trie, _ = New(common.Hash{}, root, db)
trie.tracer = newTracer()
// Delete all the elements, check deletion set
for _, val := range vals {