mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-24 13:46:43 +00:00
trie, core: track the original value of modified trie nodes
This commit is contained in:
parent
62a151e688
commit
62ddb72816
9 changed files with 246 additions and 122 deletions
|
|
@ -977,7 +977,7 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash,
|
||||||
storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot
|
storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot
|
||||||
)
|
)
|
||||||
stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) {
|
stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) {
|
||||||
nodes.AddNode(path, trienode.NewDeleted())
|
nodes.AddNode(path, trienode.NewDeletedWithPrev(blob))
|
||||||
})
|
})
|
||||||
for iter.Next() {
|
for iter.Next() {
|
||||||
slot := common.CopyBytes(iter.Slot())
|
slot := common.CopyBytes(iter.Slot())
|
||||||
|
|
@ -1028,7 +1028,7 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
|
||||||
if it.Hash() == (common.Hash{}) {
|
if it.Hash() == (common.Hash{}) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
nodes.AddNode(it.Path(), trienode.NewDeleted())
|
nodes.AddNode(it.Path(), trienode.NewDeletedWithPrev(it.NodeBlob()))
|
||||||
}
|
}
|
||||||
if err := it.Error(); err != nil {
|
if err := it.Error(); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
|
|
@ -1160,7 +1160,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool, noStorageWiping bool) (*stateU
|
||||||
//
|
//
|
||||||
// Given that some accounts may be destroyed and then recreated within
|
// Given that some accounts may be destroyed and then recreated within
|
||||||
// the same block, it's possible that a node set with the same owner
|
// the same block, it's possible that a node set with the same owner
|
||||||
// may already exists. In such cases, these two sets are combined, with
|
// may already exist. In such cases, these two sets are combined, with
|
||||||
// the later one overwriting the previous one if any nodes are modified
|
// the later one overwriting the previous one if any nodes are modified
|
||||||
// or deleted in both sets.
|
// or deleted in both sets.
|
||||||
//
|
//
|
||||||
|
|
|
||||||
|
|
@ -110,14 +110,16 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) {
|
||||||
} else {
|
} else {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func(index int) {
|
go func(index int) {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
p := append(path, byte(index))
|
p := append(path, byte(index))
|
||||||
childSet := trienode.NewNodeSet(c.nodes.Owner)
|
childSet := trienode.NewNodeSet(c.nodes.Owner)
|
||||||
childCommitter := newCommitter(childSet, c.tracer, c.collectLeaf)
|
childCommitter := newCommitter(childSet, c.tracer, c.collectLeaf)
|
||||||
n.Children[index] = childCommitter.commit(p, child, false)
|
n.Children[index] = childCommitter.commit(p, child, false)
|
||||||
|
|
||||||
nodesMu.Lock()
|
nodesMu.Lock()
|
||||||
c.nodes.MergeSet(childSet)
|
c.nodes.MergeDisjoint(childSet)
|
||||||
nodesMu.Unlock()
|
nodesMu.Unlock()
|
||||||
wg.Done()
|
|
||||||
}(i)
|
}(i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -140,14 +142,15 @@ func (c *committer) store(path []byte, n node) node {
|
||||||
// The node is embedded in its parent, in other words, this node
|
// The node is embedded in its parent, in other words, this node
|
||||||
// will not be stored in the database independently, mark it as
|
// will not be stored in the database independently, mark it as
|
||||||
// deleted only if the node was existent in database before.
|
// deleted only if the node was existent in database before.
|
||||||
if c.tracer.has(path) {
|
origin := c.tracer.get(path)
|
||||||
c.nodes.AddNode(path, trienode.NewDeleted())
|
if len(origin) != 0 {
|
||||||
|
c.nodes.AddNode(path, trienode.NewDeletedWithPrev(origin))
|
||||||
}
|
}
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
// Collect the dirty node to nodeset for return.
|
// Collect the dirty node to nodeset for return.
|
||||||
nhash := common.BytesToHash(hash)
|
nhash := common.BytesToHash(hash)
|
||||||
c.nodes.AddNode(path, trienode.New(nhash, nodeToBytes(n)))
|
c.nodes.AddNode(path, trienode.NewNodeWithPrev(nhash, nodeToBytes(n), c.tracer.get(path)))
|
||||||
|
|
||||||
// Collect the corresponding leaf node if it's required. We don't check
|
// Collect the corresponding leaf node if it's required. We don't check
|
||||||
// full node since it's impossible to store value in fullNode. The key
|
// full node since it's impossible to store value in fullNode. The key
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ package trie
|
||||||
import (
|
import (
|
||||||
"maps"
|
"maps"
|
||||||
"slices"
|
"slices"
|
||||||
"sync"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// opTracer tracks the changes of trie nodes. During the trie operations,
|
// opTracer tracks the changes of trie nodes. During the trie operations,
|
||||||
|
|
@ -96,13 +95,13 @@ func (t *opTracer) deletedList() [][]byte {
|
||||||
}
|
}
|
||||||
|
|
||||||
// prevalueTracer tracks the original values of resolved trie nodes. Cached trie
|
// prevalueTracer tracks the original values of resolved trie nodes. Cached trie
|
||||||
// node values are expected to be immutable.
|
// node values are expected to be immutable. A zero-size node value is treated as
|
||||||
|
// non-existent and should not occur in practice.
|
||||||
//
|
//
|
||||||
// prevalueTracer is protected by a lock to ensure concurrency safety, as
|
// Note prevalueTracer is not thread-safe, callers should be responsible for
|
||||||
// concurrent map writes may occur during concurrent trie read operations.
|
// handling the concurrency issues by themselves.
|
||||||
type prevalueTracer struct {
|
type prevalueTracer struct {
|
||||||
data map[string][]byte
|
data map[string][]byte
|
||||||
lock sync.RWMutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newPrevalueTracer initializes the tracer for capturing resolved trie nodes.
|
// newPrevalueTracer initializes the tracer for capturing resolved trie nodes.
|
||||||
|
|
@ -116,28 +115,18 @@ func newPrevalueTracer() *prevalueTracer {
|
||||||
// blob internally. Do not modify the value outside this function,
|
// blob internally. Do not modify the value outside this function,
|
||||||
// as it is not deep-copied.
|
// as it is not deep-copied.
|
||||||
func (t *prevalueTracer) put(path []byte, val []byte) {
|
func (t *prevalueTracer) put(path []byte, val []byte) {
|
||||||
t.lock.Lock()
|
|
||||||
defer t.lock.Unlock()
|
|
||||||
|
|
||||||
t.data[string(path)] = val
|
t.data[string(path)] = val
|
||||||
}
|
}
|
||||||
|
|
||||||
// has returns a flag indicating whether the corresponding trie node specified
|
// get returns the cached trie node value. If the node is not found, nil will
|
||||||
// by the path exists in the trie.
|
// be returned.
|
||||||
func (t *prevalueTracer) has(path []byte) bool {
|
func (t *prevalueTracer) get(path []byte) []byte {
|
||||||
t.lock.RLock()
|
return t.data[string(path)]
|
||||||
defer t.lock.RUnlock()
|
|
||||||
|
|
||||||
_, ok := t.data[string(path)]
|
|
||||||
return ok
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// hasList returns a list of flag indicating whether the corresponding trie nodes
|
// hasList returns a list of flags indicating whether the corresponding trie nodes
|
||||||
// specified by the path exists in the trie.
|
// specified by the path exist in the trie.
|
||||||
func (t *prevalueTracer) hasList(list [][]byte) []bool {
|
func (t *prevalueTracer) hasList(list [][]byte) []bool {
|
||||||
t.lock.RLock()
|
|
||||||
defer t.lock.RUnlock()
|
|
||||||
|
|
||||||
exists := make([]bool, 0, len(list))
|
exists := make([]bool, 0, len(list))
|
||||||
for _, path := range list {
|
for _, path := range list {
|
||||||
_, ok := t.data[string(path)]
|
_, ok := t.data[string(path)]
|
||||||
|
|
@ -148,25 +137,16 @@ func (t *prevalueTracer) hasList(list [][]byte) []bool {
|
||||||
|
|
||||||
// values returns a list of values of the cached trie nodes.
|
// values returns a list of values of the cached trie nodes.
|
||||||
func (t *prevalueTracer) values() [][]byte {
|
func (t *prevalueTracer) values() [][]byte {
|
||||||
t.lock.RLock()
|
|
||||||
defer t.lock.RUnlock()
|
|
||||||
|
|
||||||
return slices.Collect(maps.Values(t.data))
|
return slices.Collect(maps.Values(t.data))
|
||||||
}
|
}
|
||||||
|
|
||||||
// reset resets the cached content in the prevalueTracer.
|
// reset resets the cached content in the prevalueTracer.
|
||||||
func (t *prevalueTracer) reset() {
|
func (t *prevalueTracer) reset() {
|
||||||
t.lock.Lock()
|
|
||||||
defer t.lock.Unlock()
|
|
||||||
|
|
||||||
clear(t.data)
|
clear(t.data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy returns a copied prevalueTracer instance.
|
// copy returns a copied prevalueTracer instance.
|
||||||
func (t *prevalueTracer) copy() *prevalueTracer {
|
func (t *prevalueTracer) copy() *prevalueTracer {
|
||||||
t.lock.RLock()
|
|
||||||
defer t.lock.RUnlock()
|
|
||||||
|
|
||||||
// Shadow clone is used, as the cached trie node values are immutable
|
// Shadow clone is used, as the cached trie node values are immutable
|
||||||
return &prevalueTracer{
|
return &prevalueTracer{
|
||||||
data: maps.Clone(t.data),
|
data: maps.Clone(t.data),
|
||||||
|
|
|
||||||
|
|
@ -52,15 +52,15 @@ var (
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTrieTracer(t *testing.T) {
|
func TestTrieOpTracer(t *testing.T) {
|
||||||
testTrieTracer(t, tiny)
|
testTrieOpTracer(t, tiny)
|
||||||
testTrieTracer(t, nonAligned)
|
testTrieOpTracer(t, nonAligned)
|
||||||
testTrieTracer(t, standard)
|
testTrieOpTracer(t, standard)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tests if the trie diffs are tracked correctly. Tracer should capture
|
// Tests if the trie diffs are tracked correctly. Tracer should capture
|
||||||
// all non-leaf dirty nodes, no matter the node is embedded or not.
|
// all non-leaf dirty nodes, no matter the node is embedded or not.
|
||||||
func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
|
func testTrieOpTracer(t *testing.T, vals []struct{ k, v string }) {
|
||||||
db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
||||||
trie := NewEmpty(db)
|
trie := NewEmpty(db)
|
||||||
|
|
||||||
|
|
@ -70,6 +70,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
|
||||||
}
|
}
|
||||||
insertSet := copySet(trie.opTracer.inserts) // copy before commit
|
insertSet := copySet(trie.opTracer.inserts) // copy before commit
|
||||||
deleteSet := copySet(trie.opTracer.deletes) // copy before commit
|
deleteSet := copySet(trie.opTracer.deletes) // copy before commit
|
||||||
|
|
||||||
root, nodes := trie.Commit(false)
|
root, nodes := trie.Commit(false)
|
||||||
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
|
db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes))
|
||||||
|
|
||||||
|
|
@ -97,13 +98,13 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
|
||||||
|
|
||||||
// Test that after inserting a new batch of nodes and deleting them immediately,
|
// Test that after inserting a new batch of nodes and deleting them immediately,
|
||||||
// the trie tracer should be cleared normally as no operation happened.
|
// the trie tracer should be cleared normally as no operation happened.
|
||||||
func TestTrieTracerNoop(t *testing.T) {
|
func TestTrieOpTracerNoop(t *testing.T) {
|
||||||
testTrieTracerNoop(t, tiny)
|
testTrieOpTracerNoop(t, tiny)
|
||||||
testTrieTracerNoop(t, nonAligned)
|
testTrieOpTracerNoop(t, nonAligned)
|
||||||
testTrieTracerNoop(t, standard)
|
testTrieOpTracerNoop(t, standard)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) {
|
func testTrieOpTracerNoop(t *testing.T, vals []struct{ k, v string }) {
|
||||||
db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
||||||
trie := NewEmpty(db)
|
trie := NewEmpty(db)
|
||||||
for _, val := range vals {
|
for _, val := range vals {
|
||||||
|
|
@ -120,14 +121,14 @@ func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tests if the accessList is correctly tracked.
|
// Tests if the original value of trie nodes are correctly tracked.
|
||||||
func TestAccessList(t *testing.T) {
|
func TestPrevalueTracer(t *testing.T) {
|
||||||
testAccessList(t, tiny)
|
testPrevalueTracer(t, tiny)
|
||||||
testAccessList(t, nonAligned)
|
testPrevalueTracer(t, nonAligned)
|
||||||
testAccessList(t, standard)
|
testPrevalueTracer(t, standard)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testAccessList(t *testing.T, vals []struct{ k, v string }) {
|
func testPrevalueTracer(t *testing.T, vals []struct{ k, v string }) {
|
||||||
var (
|
var (
|
||||||
db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
||||||
trie = NewEmpty(db)
|
trie = NewEmpty(db)
|
||||||
|
|
@ -210,7 +211,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tests origin values won't be tracked in Iterator or Prover
|
// Tests origin values won't be tracked in Iterator or Prover
|
||||||
func TestAccessListLeak(t *testing.T) {
|
func TestPrevalueTracerLeak(t *testing.T) {
|
||||||
var (
|
var (
|
||||||
db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
db = newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme)
|
||||||
trie = NewEmpty(db)
|
trie = NewEmpty(db)
|
||||||
|
|
|
||||||
16
trie/trie.go
16
trie/trie.go
|
|
@ -55,7 +55,7 @@ type Trie struct {
|
||||||
// reader is the handler trie can retrieve nodes from.
|
// reader is the handler trie can retrieve nodes from.
|
||||||
reader *trieReader
|
reader *trieReader
|
||||||
|
|
||||||
// Various tracers for capturing the modification to trie
|
// Various tracers for capturing the modifications to trie
|
||||||
opTracer *opTracer
|
opTracer *opTracer
|
||||||
prevalueTracer *prevalueTracer
|
prevalueTracer *prevalueTracer
|
||||||
}
|
}
|
||||||
|
|
@ -627,18 +627,16 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// deletedNodes returns a list of node paths, referring the nodes being deleted
|
// deletedNodes returns a list of node paths, referring the nodes being deleted
|
||||||
// from the trie.
|
// from the trie. It's possible a few deleted nodes were embedded in their parent
|
||||||
|
// before, the deletions can be no effect by deleting nothing, filter them out.
|
||||||
func (t *Trie) deletedNodes() [][]byte {
|
func (t *Trie) deletedNodes() [][]byte {
|
||||||
list := t.opTracer.deletedList()
|
|
||||||
|
|
||||||
// It's possible a few deleted nodes were embedded in their parent before,
|
|
||||||
// the deletions can be no effect by deleting nothing, filter them out.
|
|
||||||
var (
|
var (
|
||||||
pos int
|
pos int
|
||||||
|
list = t.opTracer.deletedList()
|
||||||
flags = t.prevalueTracer.hasList(list)
|
flags = t.prevalueTracer.hasList(list)
|
||||||
)
|
)
|
||||||
for i := 0; i < len(list); i++ {
|
for i := 0; i < len(list); i++ {
|
||||||
if flags[i] { // exist, keep it
|
if flags[i] {
|
||||||
list[pos] = list[i]
|
list[pos] = list[i]
|
||||||
pos++
|
pos++
|
||||||
}
|
}
|
||||||
|
|
@ -673,7 +671,7 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
|
||||||
}
|
}
|
||||||
nodes := trienode.NewNodeSet(t.owner)
|
nodes := trienode.NewNodeSet(t.owner)
|
||||||
for _, path := range paths {
|
for _, path := range paths {
|
||||||
nodes.AddNode(path, trienode.NewDeleted())
|
nodes.AddNode(path, trienode.NewDeletedWithPrev(t.prevalueTracer.get(path)))
|
||||||
}
|
}
|
||||||
return types.EmptyRootHash, nodes // case (b)
|
return types.EmptyRootHash, nodes // case (b)
|
||||||
}
|
}
|
||||||
|
|
@ -691,7 +689,7 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
|
||||||
}
|
}
|
||||||
nodes := trienode.NewNodeSet(t.owner)
|
nodes := trienode.NewNodeSet(t.owner)
|
||||||
for _, path := range t.deletedNodes() {
|
for _, path := range t.deletedNodes() {
|
||||||
nodes.AddNode(path, trienode.NewDeleted())
|
nodes.AddNode(path, trienode.NewDeletedWithPrev(t.prevalueTracer.get(path)))
|
||||||
}
|
}
|
||||||
// If the number of changes is below 100, we let one thread handle it
|
// If the number of changes is below 100, we let one thread handle it
|
||||||
t.root = newCommitter(nodes, t.prevalueTracer, collectLeaf).Commit(t.root, t.uncommitted > 100)
|
t.root = newCommitter(nodes, t.prevalueTracer, collectLeaf).Commit(t.root, t.uncommitted > 100)
|
||||||
|
|
|
||||||
|
|
@ -449,35 +449,35 @@ func verifyAccessList(old *Trie, new *Trie, set *trienode.NodeSet) error {
|
||||||
if !ok || n.IsDeleted() {
|
if !ok || n.IsDeleted() {
|
||||||
return errors.New("expect new node")
|
return errors.New("expect new node")
|
||||||
}
|
}
|
||||||
//if len(n.Prev) > 0 {
|
if len(set.Origins[path]) > 0 {
|
||||||
// return errors.New("unexpected origin value")
|
return errors.New("unexpected origin value")
|
||||||
//}
|
}
|
||||||
}
|
}
|
||||||
// Check deletion set
|
// Check deletion set
|
||||||
for path := range deletes {
|
for path, blob := range deletes {
|
||||||
n, ok := set.Nodes[path]
|
n, ok := set.Nodes[path]
|
||||||
if !ok || !n.IsDeleted() {
|
if !ok || !n.IsDeleted() {
|
||||||
return errors.New("expect deleted node")
|
return errors.New("expect deleted node")
|
||||||
}
|
}
|
||||||
//if len(n.Prev) == 0 {
|
if len(set.Origins[path]) == 0 {
|
||||||
// return errors.New("expect origin value")
|
return errors.New("expect origin value")
|
||||||
//}
|
}
|
||||||
//if !bytes.Equal(n.Prev, blob) {
|
if !bytes.Equal(set.Origins[path], blob) {
|
||||||
// return errors.New("invalid origin value")
|
return errors.New("invalid origin value")
|
||||||
//}
|
}
|
||||||
}
|
}
|
||||||
// Check update set
|
// Check update set
|
||||||
for path := range updates {
|
for path, blob := range updates {
|
||||||
n, ok := set.Nodes[path]
|
n, ok := set.Nodes[path]
|
||||||
if !ok || n.IsDeleted() {
|
if !ok || n.IsDeleted() {
|
||||||
return errors.New("expect updated node")
|
return errors.New("expect updated node")
|
||||||
}
|
}
|
||||||
//if len(n.Prev) == 0 {
|
if len(set.Origins[path]) == 0 {
|
||||||
// return errors.New("expect origin value")
|
return errors.New("expect origin value")
|
||||||
//}
|
}
|
||||||
//if !bytes.Equal(n.Prev, blob) {
|
if !bytes.Equal(set.Origins[path], blob) {
|
||||||
// return errors.New("invalid origin value")
|
return errors.New("invalid origin value")
|
||||||
//}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,35 @@ func New(hash common.Hash, blob []byte) *Node {
|
||||||
// NewDeleted constructs a node which is deleted.
|
// NewDeleted constructs a node which is deleted.
|
||||||
func NewDeleted() *Node { return New(common.Hash{}, nil) }
|
func NewDeleted() *Node { return New(common.Hash{}, nil) }
|
||||||
|
|
||||||
|
// NodeWithPrev is a wrapper over Node by tracking the original value of node.
|
||||||
|
type NodeWithPrev struct {
|
||||||
|
*Node
|
||||||
|
Prev []byte // Nil means the node was not existent
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNodeWithPrev constructs a node with the additional original value.
|
||||||
|
func NewNodeWithPrev(hash common.Hash, blob []byte, prev []byte) *NodeWithPrev {
|
||||||
|
return &NodeWithPrev{
|
||||||
|
Node: &Node{
|
||||||
|
Hash: hash,
|
||||||
|
Blob: blob,
|
||||||
|
},
|
||||||
|
Prev: prev,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDeletedWithPrev constructs a node which is deleted with the additional
|
||||||
|
// original value.
|
||||||
|
func NewDeletedWithPrev(prev []byte) *NodeWithPrev {
|
||||||
|
return &NodeWithPrev{
|
||||||
|
Node: &Node{
|
||||||
|
Hash: common.Hash{},
|
||||||
|
Blob: nil,
|
||||||
|
},
|
||||||
|
Prev: prev,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// leaf represents a trie leaf node
|
// leaf represents a trie leaf node
|
||||||
type leaf struct {
|
type leaf struct {
|
||||||
Blob []byte // raw blob of leaf
|
Blob []byte // raw blob of leaf
|
||||||
|
|
@ -63,6 +92,8 @@ type NodeSet struct {
|
||||||
Owner common.Hash
|
Owner common.Hash
|
||||||
Leaves []*leaf
|
Leaves []*leaf
|
||||||
Nodes map[string]*Node
|
Nodes map[string]*Node
|
||||||
|
Origins map[string][]byte
|
||||||
|
|
||||||
updates int // the count of updated and inserted nodes
|
updates int // the count of updated and inserted nodes
|
||||||
deletes int // the count of deleted nodes
|
deletes int // the count of deleted nodes
|
||||||
}
|
}
|
||||||
|
|
@ -73,6 +104,7 @@ func NewNodeSet(owner common.Hash) *NodeSet {
|
||||||
return &NodeSet{
|
return &NodeSet{
|
||||||
Owner: owner,
|
Owner: owner,
|
||||||
Nodes: make(map[string]*Node),
|
Nodes: make(map[string]*Node),
|
||||||
|
Origins: make(map[string][]byte),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -91,22 +123,25 @@ func (set *NodeSet) ForEachWithOrder(callback func(path string, n *Node)) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddNode adds the provided node into set.
|
// AddNode adds the provided node into set.
|
||||||
func (set *NodeSet) AddNode(path []byte, n *Node) {
|
func (set *NodeSet) AddNode(path []byte, n *NodeWithPrev) {
|
||||||
if n.IsDeleted() {
|
if n.IsDeleted() {
|
||||||
set.deletes += 1
|
set.deletes += 1
|
||||||
} else {
|
} else {
|
||||||
set.updates += 1
|
set.updates += 1
|
||||||
}
|
}
|
||||||
set.Nodes[string(path)] = n
|
key := string(path)
|
||||||
|
set.Nodes[key] = n.Node
|
||||||
|
set.Origins[key] = n.Prev
|
||||||
}
|
}
|
||||||
|
|
||||||
// MergeSet merges this 'set' with 'other'. It assumes that the sets are disjoint,
|
// MergeDisjoint merges this 'set' with 'other'. It assumes that the sets are disjoint,
|
||||||
// and thus does not deduplicate data (count deletes, dedup leaves etc).
|
// and thus does not deduplicate data (count deletes, dedup leaves etc).
|
||||||
func (set *NodeSet) MergeSet(other *NodeSet) error {
|
func (set *NodeSet) MergeDisjoint(other *NodeSet) error {
|
||||||
if set.Owner != other.Owner {
|
if set.Owner != other.Owner {
|
||||||
return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, other.Owner)
|
return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, other.Owner)
|
||||||
}
|
}
|
||||||
maps.Copy(set.Nodes, other.Nodes)
|
maps.Copy(set.Nodes, other.Nodes)
|
||||||
|
maps.Copy(set.Origins, other.Origins)
|
||||||
|
|
||||||
set.deletes += other.deletes
|
set.deletes += other.deletes
|
||||||
set.updates += other.updates
|
set.updates += other.updates
|
||||||
|
|
@ -117,12 +152,13 @@ func (set *NodeSet) MergeSet(other *NodeSet) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Merge adds a set of nodes into the set.
|
// Merge adds a set of nodes to the current set. It assumes the sets may overlap,
|
||||||
func (set *NodeSet) Merge(owner common.Hash, nodes map[string]*Node) error {
|
// so deduplication is performed.
|
||||||
if set.Owner != owner {
|
func (set *NodeSet) Merge(other *NodeSet) error {
|
||||||
return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, owner)
|
if set.Owner != other.Owner {
|
||||||
|
return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, other.Owner)
|
||||||
}
|
}
|
||||||
for path, node := range nodes {
|
for path, node := range other.Nodes {
|
||||||
prev, ok := set.Nodes[path]
|
prev, ok := set.Nodes[path]
|
||||||
if ok {
|
if ok {
|
||||||
// overwrite happens, revoke the counter
|
// overwrite happens, revoke the counter
|
||||||
|
|
@ -137,8 +173,17 @@ func (set *NodeSet) Merge(owner common.Hash, nodes map[string]*Node) error {
|
||||||
} else {
|
} else {
|
||||||
set.updates += 1
|
set.updates += 1
|
||||||
}
|
}
|
||||||
set.Nodes[path] = node
|
set.Nodes[path] = node // overwrite the node with new value
|
||||||
|
|
||||||
|
// Add the original value only if it was previously non-existent.
|
||||||
|
// If multiple mutations are made to the same node, the first one
|
||||||
|
// is considered the true original value.
|
||||||
|
if _, exist := set.Origins[path]; !exist {
|
||||||
|
set.Origins[path] = other.Origins[path]
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
// TODO leaves are not aggregated, as they are not used in storage tries.
|
||||||
|
// TODO(rjl493456442) deprecate the leaves along with the legacy hash mode.
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -169,11 +214,16 @@ func (set *NodeSet) Summary() string {
|
||||||
for path, n := range set.Nodes {
|
for path, n := range set.Nodes {
|
||||||
// Deletion
|
// Deletion
|
||||||
if n.IsDeleted() {
|
if n.IsDeleted() {
|
||||||
fmt.Fprintf(out, " [-]: %x\n", path)
|
fmt.Fprintf(out, " [-]: %x prev: %x\n", path, set.Origins[path])
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// Insertion or update
|
// Insertion
|
||||||
fmt.Fprintf(out, " [+/*]: %x -> %v \n", path, n.Hash)
|
if len(set.Origins[path]) == 0 {
|
||||||
|
fmt.Fprintf(out, " [+]: %x -> %v\n", path, n.Hash)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Update
|
||||||
|
fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", path, n.Hash, set.Origins[path])
|
||||||
}
|
}
|
||||||
for _, n := range set.Leaves {
|
for _, n := range set.Leaves {
|
||||||
fmt.Fprintf(out, "[leaf]: %v\n", n)
|
fmt.Fprintf(out, "[leaf]: %v\n", n)
|
||||||
|
|
@ -203,7 +253,7 @@ func NewWithNodeSet(set *NodeSet) *MergedNodeSet {
|
||||||
func (set *MergedNodeSet) Merge(other *NodeSet) error {
|
func (set *MergedNodeSet) Merge(other *NodeSet) error {
|
||||||
subset, present := set.Sets[other.Owner]
|
subset, present := set.Sets[other.Owner]
|
||||||
if present {
|
if present {
|
||||||
return subset.Merge(other.Owner, other.Nodes)
|
return subset.Merge(other)
|
||||||
}
|
}
|
||||||
set.Sets[other.Owner] = other
|
set.Sets[other.Owner] = other
|
||||||
return nil
|
return nil
|
||||||
|
|
|
||||||
|
|
@ -17,13 +17,100 @@
|
||||||
package trienode
|
package trienode
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
|
"maps"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/davecgh/go-spew/spew"
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
|
"github.com/ethereum/go-ethereum/internal/testrand"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func makeTestSet(owner common.Hash, n int, paths [][]byte) *NodeSet {
|
||||||
|
set := NewNodeSet(owner)
|
||||||
|
for i := 0; i < n*3/4; i++ {
|
||||||
|
path := testrand.Bytes(10)
|
||||||
|
blob := testrand.Bytes(100)
|
||||||
|
set.AddNode(path, NewNodeWithPrev(crypto.Keccak256Hash(blob), blob, testrand.Bytes(100)))
|
||||||
|
}
|
||||||
|
for i := 0; i < n/4; i++ {
|
||||||
|
path := testrand.Bytes(10)
|
||||||
|
set.AddNode(path, NewDeletedWithPrev(testrand.Bytes(100)))
|
||||||
|
}
|
||||||
|
for i := 0; i < len(paths); i++ {
|
||||||
|
if i%3 == 0 {
|
||||||
|
set.AddNode(paths[i], NewDeletedWithPrev(testrand.Bytes(100)))
|
||||||
|
} else {
|
||||||
|
blob := testrand.Bytes(100)
|
||||||
|
set.AddNode(paths[i], NewNodeWithPrev(crypto.Keccak256Hash(blob), blob, testrand.Bytes(100)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return set
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyNodeSet(set *NodeSet) *NodeSet {
|
||||||
|
cpy := &NodeSet{
|
||||||
|
Owner: set.Owner,
|
||||||
|
Leaves: slices.Clone(set.Leaves),
|
||||||
|
updates: set.updates,
|
||||||
|
deletes: set.deletes,
|
||||||
|
Nodes: maps.Clone(set.Nodes),
|
||||||
|
Origins: maps.Clone(set.Origins),
|
||||||
|
}
|
||||||
|
return cpy
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeSetMerge(t *testing.T) {
|
||||||
|
var shared [][]byte
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
shared = append(shared, testrand.Bytes(10))
|
||||||
|
}
|
||||||
|
owner := testrand.Hash()
|
||||||
|
setA := makeTestSet(owner, 20, shared)
|
||||||
|
cpyA := copyNodeSet(setA)
|
||||||
|
|
||||||
|
setB := makeTestSet(owner, 20, shared)
|
||||||
|
setA.Merge(setB)
|
||||||
|
|
||||||
|
for path, node := range setA.Nodes {
|
||||||
|
nA, inA := cpyA.Nodes[path]
|
||||||
|
nB, inB := setB.Nodes[path]
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case inA && inB:
|
||||||
|
origin := setA.Origins[path]
|
||||||
|
if !bytes.Equal(origin, cpyA.Origins[path]) {
|
||||||
|
t.Errorf("Unexpected origin, path %v: want: %v, got: %v", []byte(path), cpyA.Origins[path], origin)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(node, nB) {
|
||||||
|
t.Errorf("Unexpected node, path %v: want: %v, got: %v", []byte(path), spew.Sdump(nB), spew.Sdump(node))
|
||||||
|
}
|
||||||
|
case !inA && inB:
|
||||||
|
origin := setA.Origins[path]
|
||||||
|
if !bytes.Equal(origin, setB.Origins[path]) {
|
||||||
|
t.Errorf("Unexpected origin, path %v: want: %v, got: %v", []byte(path), setB.Origins[path], origin)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(node, nB) {
|
||||||
|
t.Errorf("Unexpected node, path %v: want: %v, got: %v", []byte(path), spew.Sdump(nB), spew.Sdump(node))
|
||||||
|
}
|
||||||
|
case inA && !inB:
|
||||||
|
origin := setA.Origins[path]
|
||||||
|
if !bytes.Equal(origin, cpyA.Origins[path]) {
|
||||||
|
t.Errorf("Unexpected origin, path %v: want: %v, got: %v", []byte(path), cpyA.Origins[path], origin)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(node, nA) {
|
||||||
|
t.Errorf("Unexpected node, path %v: want: %v, got: %v", []byte(path), spew.Sdump(nA), spew.Sdump(node))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Errorf("Unexpected node, %v", []byte(path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func BenchmarkMerge(b *testing.B) {
|
func BenchmarkMerge(b *testing.B) {
|
||||||
b.Run("1K", func(b *testing.B) {
|
b.Run("1K", func(b *testing.B) {
|
||||||
benchmarkMerge(b, 1000)
|
benchmarkMerge(b, 1000)
|
||||||
|
|
@ -42,7 +129,7 @@ func benchmarkMerge(b *testing.B, count int) {
|
||||||
blob := make([]byte, 32)
|
blob := make([]byte, 32)
|
||||||
rand.Read(blob)
|
rand.Read(blob)
|
||||||
hash := crypto.Keccak256Hash(blob)
|
hash := crypto.Keccak256Hash(blob)
|
||||||
s.AddNode(path, New(hash, blob))
|
s.AddNode(path, NewNodeWithPrev(hash, blob, nil))
|
||||||
}
|
}
|
||||||
for i := 0; i < count; i++ {
|
for i := 0; i < count; i++ {
|
||||||
// Random path of 4 nibbles
|
// Random path of 4 nibbles
|
||||||
|
|
@ -53,9 +140,9 @@ func benchmarkMerge(b *testing.B, count int) {
|
||||||
for i := 0; i < b.N; i++ {
|
for i := 0; i < b.N; i++ {
|
||||||
// Store set x into a backup
|
// Store set x into a backup
|
||||||
z := NewNodeSet(common.Hash{})
|
z := NewNodeSet(common.Hash{})
|
||||||
z.Merge(common.Hash{}, x.Nodes)
|
z.Merge(x)
|
||||||
// Merge y into x
|
// Merge y into x
|
||||||
x.Merge(common.Hash{}, y.Nodes)
|
x.Merge(y)
|
||||||
x = z
|
x = z
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ type VerkleTrie struct {
|
||||||
root verkle.VerkleNode
|
root verkle.VerkleNode
|
||||||
cache *utils.PointCache
|
cache *utils.PointCache
|
||||||
reader *trieReader
|
reader *trieReader
|
||||||
|
tracer *prevalueTracer
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewVerkleTrie constructs a verkle tree based on the specified root hash.
|
// NewVerkleTrie constructs a verkle tree based on the specified root hash.
|
||||||
|
|
@ -50,27 +51,25 @@ func NewVerkleTrie(root common.Hash, db database.NodeDatabase, cache *utils.Poin
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Parse the root verkle node if it's not empty.
|
t := &VerkleTrie{
|
||||||
node := verkle.New()
|
root: verkle.New(),
|
||||||
if root != types.EmptyVerkleHash && root != types.EmptyRootHash {
|
|
||||||
blob, err := reader.node(nil, common.Hash{})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
node, err = verkle.ParseNode(blob, 0)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &VerkleTrie{
|
|
||||||
root: node,
|
|
||||||
cache: cache,
|
cache: cache,
|
||||||
reader: reader,
|
reader: reader,
|
||||||
}, nil
|
tracer: newPrevalueTracer(),
|
||||||
}
|
}
|
||||||
|
// Parse the root verkle node if it's not empty.
|
||||||
func (t *VerkleTrie) FlatdbNodeResolver(path []byte) ([]byte, error) {
|
if root != types.EmptyVerkleHash && root != types.EmptyRootHash {
|
||||||
return t.reader.node(path, common.Hash{})
|
blob, err := t.nodeResolver(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
node, err := verkle.ParseNode(blob, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
t.root = node
|
||||||
|
}
|
||||||
|
return t, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetKey returns the sha3 preimage of a hashed key that was previously used
|
// GetKey returns the sha3 preimage of a hashed key that was previously used
|
||||||
|
|
@ -268,7 +267,7 @@ func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) {
|
||||||
nodeset := trienode.NewNodeSet(common.Hash{})
|
nodeset := trienode.NewNodeSet(common.Hash{})
|
||||||
for _, node := range nodes {
|
for _, node := range nodes {
|
||||||
// Hash parameter is not used in pathdb
|
// Hash parameter is not used in pathdb
|
||||||
nodeset.AddNode(node.Path, trienode.New(common.Hash{}, node.SerializedBytes))
|
nodeset.AddNode(node.Path, trienode.NewNodeWithPrev(common.Hash{}, node.SerializedBytes, t.tracer.get(node.Path)))
|
||||||
}
|
}
|
||||||
// Serialize root commitment form
|
// Serialize root commitment form
|
||||||
return t.Hash(), nodeset
|
return t.Hash(), nodeset
|
||||||
|
|
@ -301,6 +300,7 @@ func (t *VerkleTrie) Copy() *VerkleTrie {
|
||||||
root: t.root.Copy(),
|
root: t.root.Copy(),
|
||||||
cache: t.cache,
|
cache: t.cache,
|
||||||
reader: t.reader,
|
reader: t.reader,
|
||||||
|
tracer: t.tracer.copy(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -317,7 +317,7 @@ func (t *VerkleTrie) Proof(posttrie *VerkleTrie, keys [][]byte) (*verkle.VerkleP
|
||||||
if posttrie != nil {
|
if posttrie != nil {
|
||||||
postroot = posttrie.root
|
postroot = posttrie.root
|
||||||
}
|
}
|
||||||
proof, _, _, _, err := verkle.MakeVerkleMultiProof(t.root, postroot, keys, t.FlatdbNodeResolver)
|
proof, _, _, _, err := verkle.MakeVerkleMultiProof(t.root, postroot, keys, t.nodeResolver)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -421,7 +421,12 @@ func (t *VerkleTrie) ToDot() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) {
|
func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) {
|
||||||
return t.reader.node(path, common.Hash{})
|
blob, err := t.reader.node(path, common.Hash{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
t.tracer.put(path, blob)
|
||||||
|
return blob, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Witness returns a set containing all trie nodes that have been accessed.
|
// Witness returns a set containing all trie nodes that have been accessed.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue