forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 02:42:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core, triedb, trie: don't track node hash in pathdb

Browse source
This commit is contained in:
Gary Rong 2023-08-16 14:58:34 +08:00
parent 14cc967d19
commit adc4b72c7e
23 changed files with 316 additions and 347 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/geth/dbcmd.go
View file

@ -246,11 +246,17 @@ func removeDB(ctx *cli.Context) error {
ancientDir = config.Node.ResolvePath(ancientDir)
}
// Delete state data
statePaths := []string{rootDir, filepath.Join(ancientDir, rawdb.StateFreezerName)}
statePaths := []string{
rootDir,
filepath.Join(ancientDir, rawdb.StateFreezerName),
}
confirmAndRemoveDB(statePaths, "state data", ctx, removeStateDataFlag.Name)
// Delete ancient chain
chainPaths := []string{filepath.Join(ancientDir, rawdb.ChainFreezerName)}
chainPaths := []string{filepath.Join(
ancientDir,
rawdb.ChainFreezerName,
)}
confirmAndRemoveDB(chainPaths, "ancient chain", ctx, removeChainDataFlag.Name)
return nil
}

2
core/genesis_test.go
View file

@ -322,7 +322,7 @@ func TestVerkleGenesisCommit(t *testing.T) {
t.Fatalf("expected trie to be verkle")
}
if !rawdb.ExistsAccountTrieNode(db, nil) {
if !rawdb.HasAccountTrieNode(db, nil) {
t.Fatal("could not find node")
}
}

124
core/rawdb/accessors_trie.go
View file

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"golang.org/x/crypto/sha3"
)
// HashScheme is the legacy hash-based state scheme with which trie nodes are
@ -50,7 +49,7 @@ const PathScheme = "path"
type hasher struct{ sha crypto.KeccakState }
var hasherPool = sync.Pool{
New: func() interface{} { return &hasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
New: func() interface{} { return &hasher{sha: crypto.NewKeccakState()} },
}
func newHasher() *hasher {
@ -65,33 +64,15 @@ func (h *hasher) release() {
hasherPool.Put(h)
}
// ReadAccountTrieNode retrieves the account trie node and the associated node
// hash with the specified node path.
func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) {
data, err := db.Get(accountTrieNodeKey(path))
if err != nil {
return nil, common.Hash{}
}
h := newHasher()
defer h.release()
return data, h.hash(data)
// ReadAccountTrieNode retrieves the account trie node with the specified node path.
func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) []byte {
data, _ := db.Get(accountTrieNodeKey(path))
return data
}
// HasAccountTrieNode checks the account trie node presence with the specified
// node path and the associated node hash.
func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool {
data, err := db.Get(accountTrieNodeKey(path))
if err != nil {
return false
}
h := newHasher()
defer h.release()
return h.hash(data) == hash
}
// ExistsAccountTrieNode checks the presence of the account trie node with the
// HasAccountTrieNode checks the presence of the account trie node with the
// specified node path, regardless of the node hash.
func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool {
func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool {
has, err := db.Has(accountTrieNodeKey(path))
if err != nil {
return false
@ -113,33 +94,15 @@ func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) {
}
}
// ReadStorageTrieNode retrieves the storage trie node and the associated node
// hash with the specified node path.
func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) {
data, err := db.Get(storageTrieNodeKey(accountHash, path))
if err != nil {
return nil, common.Hash{}
}
h := newHasher()
defer h.release()
return data, h.hash(data)
// ReadStorageTrieNode retrieves the storage trie node with the specified node path.
func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) []byte {
data, _ := db.Get(storageTrieNodeKey(accountHash, path))
return data
}
// HasStorageTrieNode checks the storage trie node presence with the provided
// node path and the associated node hash.
func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool {
data, err := db.Get(storageTrieNodeKey(accountHash, path))
if err != nil {
return false
}
h := newHasher()
defer h.release()
return h.hash(data) == hash
}
// ExistsStorageTrieNode checks the presence of the storage trie node with the
// HasStorageTrieNode checks the presence of the storage trie node with the
// specified account hash and node path, regardless of the node hash.
func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool {
func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool {
has, err := db.Has(storageTrieNodeKey(accountHash, path))
if err != nil {
return false
@ -198,10 +161,18 @@ func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash c
case HashScheme:
return HasLegacyTrieNode(db, hash)
case PathScheme:
var blob []byte
if owner == (common.Hash{}) {
return HasAccountTrieNode(db, path, hash)
blob = ReadAccountTrieNode(db, path)
} else {
blob = ReadStorageTrieNode(db, owner, path)
}
return HasStorageTrieNode(db, owner, path, hash)
if len(blob) == 0 {
return false
}
h := newHasher()
defer h.release()
return h.hash(blob) == hash // exists but not match
default:
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
@ -209,28 +180,21 @@ func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash c
// ReadTrieNode retrieves the trie node from database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
//
// pathScheme-based lookup requires the following:
// - owner
// - path
func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte {
switch scheme {
case HashScheme:
return ReadLegacyTrieNode(db, hash)
case PathScheme:
var (
blob []byte
nHash common.Hash
)
var blob []byte
if owner == (common.Hash{}) {
blob, nHash = ReadAccountTrieNode(db, path)
blob = ReadAccountTrieNode(db, path)
} else {
blob, nHash = ReadStorageTrieNode(db, owner, path)
blob = ReadStorageTrieNode(db, owner, path)
}
if nHash != hash {
return nil
h := newHasher()
defer h.release()
if h.hash(blob) != hash {
return nil // exists but not match
}
return blob
default:
@ -238,14 +202,10 @@ func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash
}
}
// WriteTrieNode writes the trie node into database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
// WriteTrieNode writes the trie node into database with the provided node info.
//
// pathScheme-based lookup requires the following:
// - owner
// - path
// hash-scheme requires the node hash as the identifier.
// path-scheme requires the node owner and path as the identifier.
func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) {
switch scheme {
case HashScheme:
@ -261,14 +221,10 @@ func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash
}
}
// DeleteTrieNode deletes the trie node from database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
// DeleteTrieNode deletes the trie node from database with the provided node info.
//
// pathScheme-based lookup requires the following:
// - owner
// - path
// hash-scheme requires the node hash as the identifier.
// path-scheme requires the node owner and path as the identifier.
func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) {
switch scheme {
case HashScheme:
@ -287,9 +243,8 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
// Check if state in path-based scheme is present
blob, _ := ReadAccountTrieNode(db, nil)
if len(blob) != 0 {
// Check if state in path-based scheme is present.
if HasAccountTrieNode(db, nil) {
return PathScheme
}
// The root node might be deleted during the initial snap sync, check
@ -304,8 +259,7 @@ func ReadStateScheme(db ethdb.Reader) string {
if header == nil {
return "" // empty datadir
}
blob = ReadLegacyTrieNode(db, header.Root)
if len(blob) == 0 {
if !HasLegacyTrieNode(db, header.Root) {
return "" // no state in disk
}
return HashScheme

3
core/rawdb/ancient_scheme.go
View file

@ -77,5 +77,6 @@ var freezers = []string{ChainFreezerName, StateFreezerName}
// NewStateFreezer initializes the freezer for state history.
func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
return NewResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
name := filepath.Join(ancientDir, StateFreezerName)
return NewResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
}

7
core/rawdb/ancient_utils.go
View file

@ -89,20 +89,17 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
infos = append(infos, info)
case StateFreezerName:
if ReadStateScheme(db) != PathScheme {
continue
}
datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
if err != nil {
return nil, err
continue // might be possible the state freezer is not existent
}
defer f.Close()
info, err := inspect(StateFreezerName, stateFreezerNoSnappy, f)
info, err := inspect(freezer, stateFreezerNoSnappy, f)
if err != nil {
return nil, err
}

19
eth/protocols/snap/sync.go
View file

@ -718,11 +718,11 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error {
// cleanPath is used to remove the dangling nodes in the stackTrie.
func (s *Syncer) cleanPath(batch ethdb.Batch, owner common.Hash, path []byte) {
if owner == (common.Hash{}) && rawdb.ExistsAccountTrieNode(s.db, path) {
if owner == (common.Hash{}) && rawdb.HasAccountTrieNode(s.db, path) {
rawdb.DeleteAccountTrieNode(batch, path)
deletionGauge.Inc(1)
}
if owner != (common.Hash{}) && rawdb.ExistsStorageTrieNode(s.db, owner, path) {
if owner != (common.Hash{}) && rawdb.HasStorageTrieNode(s.db, owner, path) {
rawdb.DeleteStorageTrieNode(batch, owner, path)
deletionGauge.Inc(1)
}
@ -2201,11 +2201,16 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
// If the chunk's root is an overflown but full delivery,
// clear the heal request.
accountHash := res.accounts[len(res.accounts)-1]
if root == res.subTask.root && rawdb.HasStorageTrieNode(s.db, accountHash, nil, root) {
for i, account := range res.mainTask.res.hashes {
if account == accountHash {
res.mainTask.needHeal[i] = false
skipStorageHealingGauge.Inc(1)
if root == res.subTask.root {
// Ensure the root node with particular hash is present in disk
// before clearing.
blob := rawdb.ReadStorageTrieNode(s.db, accountHash, nil)
if len(blob) != 0 && crypto.Keccak256Hash(blob) == root {
for i, account := range res.mainTask.res.hashes {
if account == accountHash {
res.mainTask.needHeal[i] = false
skipStorageHealingGauge.Inc(1)
}
}
}
}

34
trie/sync.go
View file

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/common/prque"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
@ -546,9 +547,9 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
// the performance impact negligible.
var exists bool
if owner == (common.Hash{}) {
exists = rawdb.ExistsAccountTrieNode(s.database, append(inner, key[:i]...))
exists = rawdb.HasAccountTrieNode(s.database, append(inner, key[:i]...))
} else {
exists = rawdb.ExistsStorageTrieNode(s.database, owner, append(inner, key[:i]...))
exists = rawdb.HasStorageTrieNode(s.database, owner, append(inner, key[:i]...))
}
if exists {
s.membatch.delNode(owner, append(inner, key[:i]...))
@ -691,13 +692,14 @@ func (s *Sync) hasNode(owner common.Hash, path []byte, hash common.Hash) (exists
}
// If node is running with path scheme, check the presence with node path.
var blob []byte
var dbHash common.Hash
if owner == (common.Hash{}) {
blob, dbHash = rawdb.ReadAccountTrieNode(s.database, path)
blob = rawdb.ReadAccountTrieNode(s.database, path)
} else {
blob, dbHash = rawdb.ReadStorageTrieNode(s.database, owner, path)
blob = rawdb.ReadStorageTrieNode(s.database, owner, path)
}
exists = hash == dbHash
h := newBlobHasher()
defer h.release()
exists = hash == h.hash(blob)
inconsistent = !exists && len(blob) != 0
return exists, inconsistent
}
@ -712,3 +714,23 @@ func ResolvePath(path []byte) (common.Hash, []byte) {
}
return owner, path
}
// blobHasher is used to compute the sha256 hash of the provided data.
type blobHasher struct{ state crypto.KeccakState }
// blobHasherPool is the pool for reusing pre-allocated hash state.
var blobHasherPool = sync.Pool{
New: func() interface{} { return &blobHasher{state: crypto.NewKeccakState()} },
}
func newBlobHasher() *blobHasher {
return blobHasherPool.Get().(*blobHasher)
}
func (h *blobHasher) hash(data []byte) common.Hash {
return crypto.HashData(h.state, data)
}
func (h *blobHasher) release() {
blobHasherPool.Put(h)
}

29
trie/trienode/node.go
View file

@ -32,11 +32,6 @@ type Node struct {
Blob []byte // Encoded node blob, nil for the deleted node
}
// Size returns the total memory size used by this node.
func (n *Node) Size() int {
return len(n.Blob) + common.HashLength
}
// IsDeleted returns the indicator if the node is marked as deleted.
func (n *Node) IsDeleted() bool {
return len(n.Blob) == 0
@ -130,16 +125,6 @@ func (set *NodeSet) Size() (int, int) {
return set.updates, set.deletes
}
// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can
// we get rid of it?
func (set *NodeSet) Hashes() []common.Hash {
var ret []common.Hash
for _, node := range set.Nodes {
ret = append(ret, node.Hash)
}
return ret
}
// Summary returns a string-representation of the NodeSet.
func (set *NodeSet) Summary() string {
var out = new(strings.Builder)
@ -189,11 +174,17 @@ func (set *MergedNodeSet) Merge(other *NodeSet) error {
return nil
}
// Flatten returns a two-dimensional map for internal nodes.
func (set *MergedNodeSet) Flatten() map[common.Hash]map[string]*Node {
nodes := make(map[common.Hash]map[string]*Node)
// Slim returns a node map by assembling nodes from the merged set while excluding
// any other fields in the MergedNodeSet. The node blobs are referenced directly
// without deep-copied due to the fact that they are immutable.
func (set *MergedNodeSet) Slim() map[common.Hash]map[string][]byte {
nodes := make(map[common.Hash]map[string][]byte)
for owner, set := range set.Sets {
nodes[owner] = set.Nodes
subset := make(map[string][]byte)
for path, node := range set.Nodes {
subset[path] = node.Blob
}
nodes[owner] = subset
}
return nodes
}

4
trie/triestate/state.go
View file

@ -102,7 +102,7 @@ type context struct {
// Apply traverses the provided state diffs, apply them in the associated
// post-state and return the generated dirty trie nodes. The state can be
// loaded via the provided trie loader.
func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, loader TrieLoader) (map[common.Hash]map[string]*trienode.Node, error) {
func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, loader TrieLoader) (map[common.Hash]map[string][]byte, error) {
tr, err := loader.OpenTrie(postRoot)
if err != nil {
return nil, err
@ -136,7 +136,7 @@ func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Addre
if err := ctx.nodes.Merge(result); err != nil {
return nil, err
}
return ctx.nodes.Flatten(), nil
return ctx.nodes.Slim(), nil
}
// updateAccount the account was present in prev-state, and may or may not

9
triedb/database.go
View file

@ -20,6 +20,7 @@ import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie"
@ -48,9 +49,6 @@ var HashDefaults = &Config{
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {
// Scheme returns the identifier of used storage scheme.
Scheme() string
// Initialized returns an indicator if the state data is already initialized
// according to the state scheme.
Initialized(genesisRoot common.Hash) bool
@ -181,7 +179,10 @@ func (db *Database) Initialized(genesisRoot common.Hash) bool {
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {
return db.backend.Scheme()
if db.config.PathDB != nil {
return rawdb.PathScheme
}
return rawdb.HashScheme
}
// Close flushes the dangling preimages to disk and closes the trie database.

3
triedb/database/database.go
View file

@ -25,6 +25,9 @@ type Reader interface {
// Node retrieves the trie node blob with the provided trie identifier,
// node path and the corresponding node hash. No error will be returned
// if the node is not found.
//
// Don't modify the returned byte slice since it's not deep-copied and
// still be referenced by database.
Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
}

5
triedb/hashdb/database.go
View file

@ -623,11 +623,6 @@ func (db *Database) Close() error {
return nil
}
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {
return rawdb.HashScheme
}
// Reader retrieves a node reader belonging to the given state root.
// An error will be returned if the requested state is not available.
func (db *Database) Reader(root common.Hash) (*reader, error) {

28
triedb/pathdb/database.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
@ -60,10 +61,11 @@ var (
// public methods and some additional methods for internal usage.
type layer interface {
// Node retrieves the trie node with the node info. An error will be returned
// if the read operation exits abnormally. For example, if the layer is already
// stale, or the associated state is regarded as corrupted. Notably, no error
// will be returned if the requested node is not found in database.
Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
// if the read operation exits abnormally. Specifically, if the layer is
// already stale.
//
// Note, no error will be returned if the requested node is not found in database.
node(owner common.Hash, path []byte, depth int) ([]byte, *nodeLoc, error)
// rootHash returns the root hash for which this layer was made.
rootHash() common.Hash
@ -78,7 +80,7 @@ type layer interface {
// the provided dirty trie nodes along with the state change set.
//
// Note, the maps are retained by the method to avoid copying everything.
update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer
update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string][]byte, states *triestate.Set) *diffLayer
// journal commits an entire diff hierarchy to disk into a single journal entry.
// This is meant to be used during shutdown to persist the layer without
@ -208,15 +210,6 @@ func New(diskdb ethdb.Database, config *Config) *Database {
return db
}
// Reader retrieves a layer belonging to the given state root.
func (db *Database) Reader(root common.Hash) (layer, error) {
l := db.tree.get(root)
if l == nil {
return nil, fmt.Errorf("state %#x is not available", root)
}
return l, nil
}
// Update adds a new layer into the tree, if that can be linked to an existing
// old parent. It is disallowed to insert a disk layer (the origin of all). Apart
// from that this function will flatten the extra diff layers at bottom into disk
@ -298,7 +291,7 @@ func (db *Database) Enable(root common.Hash) error {
}
// Ensure the provided state root matches the stored one.
root = types.TrieRootHash(root)
_, stored := rawdb.ReadAccountTrieNode(db.diskdb, nil)
stored := crypto.Keccak256Hash(rawdb.ReadAccountTrieNode(db.diskdb, nil))
if stored != root {
return fmt.Errorf("state root mismatch: stored %x, synced %x", stored, root)
}
@ -471,11 +464,6 @@ func (db *Database) SetBufferSize(size int) error {
return db.tree.bottom().setBufferSize(db.bufferSize)
}
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() string {
return rawdb.PathScheme
}
// modifyAllowed returns the indicator if mutation is allowed. This function
// assumes the db.lock is already held.
func (db *Database) modifyAllowed() error {

6
triedb/pathdb/database_test.go
View file

@ -474,7 +474,7 @@ func TestDisable(t *testing.T) {
tester := newTester(t, 0)
defer tester.release()
_, stored := rawdb.ReadAccountTrieNode(tester.db.diskdb, nil)
stored := crypto.Keccak256Hash(rawdb.ReadAccountTrieNode(tester.db.diskdb, nil))
if err := tester.db.Disable(); err != nil {
t.Fatal("Failed to deactivate database")
}
@ -580,7 +580,9 @@ func TestCorruptedJournal(t *testing.T) {
t.Errorf("Failed to journal, err: %v", err)
}
tester.db.Close()
_, root := rawdb.ReadAccountTrieNode(tester.db.diskdb, nil)
rootBlob := rawdb.ReadAccountTrieNode(tester.db.diskdb, nil)
root := crypto.Keccak256Hash(rootBlob)
// Mutate the journal in disk, it should be regarded as invalid
blob := rawdb.ReadTrieJournal(tester.db.diskdb)

51
triedb/pathdb/difflayer.go
View file

@ -22,7 +22,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/triestate"
)
@ -33,19 +32,19 @@ import (
// made to the state, that have not yet graduated into a semi-immutable state.
type diffLayer struct {
// Immutables
root common.Hash // Root hash to which this layer diff belongs to
id uint64 // Corresponding state id
block uint64 // Associated block number
nodes map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
states *triestate.Set // Associated state change set for building history
memory uint64 // Approximate guess as to how much memory we use
root common.Hash // Root hash to which this layer diff belongs to
id uint64 // Corresponding state id
block uint64 // Associated block number
nodes map[common.Hash]map[string][]byte // Cached trie nodes indexed by owner and path
states *triestate.Set // Associated state change set for building history
memory uint64 // Approximate guess as to how much memory we use
parent layer // Parent layer modified by this one, never nil, **can be changed**
lock sync.RWMutex // Lock used to protect parent
}
// newDiffLayer creates a new diff layer on top of an existing layer.
func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {
func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string][]byte, states *triestate.Set) *diffLayer {
var (
size int64
count int
@ -60,11 +59,11 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
}
for _, subset := range nodes {
for path, n := range subset {
dl.memory += uint64(n.Size() + len(path))
size += int64(len(n.Blob) + len(path))
size += int64(len(n) + len(path))
}
count += len(subset)
}
dl.memory = uint64(size)
if states != nil {
dl.memory += uint64(states.Size())
}
@ -95,10 +94,9 @@ func (dl *diffLayer) parentLayer() layer {
return dl.parent
}
// node retrieves the node with provided node information. It's the internal
// version of Node function with additional accessed layer tracked. No error
// will be returned if node is not found.
func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, depth int) ([]byte, error) {
// node implements the layer interface, retrieving the trie node blob with the
// provided node information. No error will be returned if the node is not found.
func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, *nodeLoc, error) {
// Hold the lock, ensure the parent won't be changed during the
// state accessing.
dl.lock.RLock()
@ -109,36 +107,19 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
if ok {
n, ok := subset[string(path)]
if ok {
// If the trie node is not hash matched, or marked as removed,
// bubble up an error here. It shouldn't happen at all.
if n.Hash != hash {
dirtyFalseMeter.Mark(1)
log.Error("Unexpected trie node in diff layer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
return nil, newUnexpectedNodeError("diff", hash, n.Hash, owner, path, n.Blob)
}
dirtyHitMeter.Mark(1)
dirtyNodeHitDepthHist.Update(int64(depth))
dirtyReadMeter.Mark(int64(len(n.Blob)))
return n.Blob, nil
dirtyReadMeter.Mark(int64(len(n)))
return n, &nodeLoc{loc: locDiffLayer, depth: depth}, nil
}
}
// Trie node unknown to this layer, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.node(owner, path, hash, depth+1)
}
// Failed to resolve through diff layers, fallback to disk layer
return dl.parent.Node(owner, path, hash)
}
// Node implements the layer interface, retrieving the trie node blob with the
// provided node information. No error will be returned if the node is not found.
func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
return dl.node(owner, path, hash, 0)
return dl.parent.node(owner, path, depth+1)
}
// update implements the layer interface, creating a new layer on top of the
// existing layer tree with the specified data items.
func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {
func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string][]byte, states *triestate.Set) *diffLayer {
return newDiffLayer(dl, root, id, block, nodes, states)
}

29
triedb/pathdb/difflayer_test.go
View file

@ -22,9 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/testrand"
"github.com/ethereum/go-ethereum/trie/trienode"
)
func emptyLayer() *diskLayer {
@ -58,24 +56,21 @@ func BenchmarkSearch1Layer(b *testing.B) { benchmarkSearch(b, 127, 128) }
func benchmarkSearch(b *testing.B, depth int, total int) {
var (
npath []byte
nhash common.Hash
nblob []byte
)
// First, we set up 128 diff layers, with 3K items each
fill := func(parent layer, index int) *diffLayer {
nodes := make(map[common.Hash]map[string]*trienode.Node)
nodes[common.Hash{}] = make(map[string]*trienode.Node)
nodes := make(map[common.Hash]map[string][]byte)
nodes[common.Hash{}] = make(map[string][]byte)
for i := 0; i < 3000; i++ {
var (
path = testrand.Bytes(32)
blob = testrand.Bytes(100)
node = trienode.New(crypto.Keccak256Hash(blob), blob)
)
nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
nodes[common.Hash{}][string(path)] = blob
if npath == nil && depth == index {
npath = common.CopyBytes(path)
nblob = common.CopyBytes(node.Blob)
nhash = node.Hash
nblob = common.CopyBytes(blob)
}
}
return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil)
@ -92,7 +87,7 @@ func benchmarkSearch(b *testing.B, depth int, total int) {
err error
)
for i := 0; i < b.N; i++ {
have, err = layer.Node(common.Hash{}, npath, nhash)
have, _, err = layer.node(common.Hash{}, npath, 0)
if err != nil {
b.Fatal(err)
}
@ -110,15 +105,14 @@ func benchmarkSearch(b *testing.B, depth int, total int) {
func BenchmarkPersist(b *testing.B) {
// First, we set up 128 diff layers, with 3K items each
fill := func(parent layer) *diffLayer {
nodes := make(map[common.Hash]map[string]*trienode.Node)
nodes[common.Hash{}] = make(map[string]*trienode.Node)
nodes := make(map[common.Hash]map[string][]byte)
nodes[common.Hash{}] = make(map[string][]byte)
for i := 0; i < 3000; i++ {
var (
path = testrand.Bytes(32)
blob = testrand.Bytes(100)
node = trienode.New(crypto.Keccak256Hash(blob), blob)
)
nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
nodes[common.Hash{}][string(path)] = blob
}
return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil)
}
@ -148,15 +142,14 @@ func BenchmarkJournal(b *testing.B) {
// First, we set up 128 diff layers, with 3K items each
fill := func(parent layer) *diffLayer {
nodes := make(map[common.Hash]map[string]*trienode.Node)
nodes[common.Hash{}] = make(map[string]*trienode.Node)
nodes := make(map[common.Hash]map[string][]byte)
nodes[common.Hash{}] = make(map[string][]byte)
for i := 0; i < 3000; i++ {
var (
path = testrand.Bytes(32)
blob = testrand.Bytes(100)
node = trienode.New(crypto.Keccak256Hash(blob), blob)
)
nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob)
nodes[common.Hash{}][string(path)] = blob
}
// TODO(rjl493456442) a non-nil state set is expected.
return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil)

78
triedb/pathdb/disklayer.go
View file

@ -23,11 +23,8 @@ import (
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/triestate"
"golang.org/x/crypto/sha3"
)
// diskLayer is a low level persistent layer built on top of a key-value store.
@ -94,27 +91,25 @@ func (dl *diskLayer) markStale() {
dl.stale = true
}
// Node implements the layer interface, retrieving the trie node with the
// node implements the layer interface, retrieving the trie node with the
// provided node info. No error will be returned if the node is not found.
func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, *nodeLoc, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.stale {
return nil, errSnapshotStale
return nil, nil, errSnapshotStale
}
// Try to retrieve the trie node from the not-yet-written
// node buffer first. Note the buffer is lock free since
// it's impossible to mutate the buffer before tagging the
// layer as stale.
n, err := dl.buffer.node(owner, path, hash)
if err != nil {
return nil, err
}
if n != nil {
n, found := dl.buffer.node(owner, path)
if found {
dirtyHitMeter.Mark(1)
dirtyReadMeter.Mark(int64(len(n.Blob)))
return n.Blob, nil
dirtyReadMeter.Mark(int64(len(n)))
dirtyNodeHitDepthHist.Update(int64(depth))
return n, &nodeLoc{loc: locDirtyCache, depth: depth}, nil
}
dirtyMissMeter.Mark(1)
@ -122,45 +117,29 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
key := cacheKey(owner, path)
if dl.cleans != nil {
if blob := dl.cleans.Get(nil, key); len(blob) > 0 {
h := newHasher()
defer h.release()
got := h.hash(blob)
if got == hash {
cleanHitMeter.Mark(1)
cleanReadMeter.Mark(int64(len(blob)))
return blob, nil
}
cleanFalseMeter.Mark(1)
log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
cleanHitMeter.Mark(1)
cleanReadMeter.Mark(int64(len(blob)))
return blob, &nodeLoc{loc: locCleanCache, depth: depth}, nil
}
cleanMissMeter.Mark(1)
}
// Try to retrieve the trie node from the disk.
var (
nBlob []byte
nHash common.Hash
)
var blob []byte
if owner == (common.Hash{}) {
nBlob, nHash = rawdb.ReadAccountTrieNode(dl.db.diskdb, path)
blob = rawdb.ReadAccountTrieNode(dl.db.diskdb, path)
} else {
nBlob, nHash = rawdb.ReadStorageTrieNode(dl.db.diskdb, owner, path)
blob = rawdb.ReadStorageTrieNode(dl.db.diskdb, owner, path)
}
if nHash != hash {
diskFalseMeter.Mark(1)
log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
if dl.cleans != nil && len(blob) > 0 {
dl.cleans.Set(key, blob)
cleanWriteMeter.Mark(int64(len(blob)))
}
if dl.cleans != nil && len(nBlob) > 0 {
dl.cleans.Set(key, nBlob)
cleanWriteMeter.Mark(int64(len(nBlob)))
}
return nBlob, nil
return blob, &nodeLoc{loc: locDiskLayer, depth: depth}, nil
}
// update implements the layer interface, returning a new diff layer on top
// with the given state set.
func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string]*trienode.Node, states *triestate.Set) *diffLayer {
func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes map[common.Hash]map[string][]byte, states *triestate.Set) *diffLayer {
return newDiffLayer(dl, root, id, block, nodes, states)
}
@ -310,22 +289,3 @@ func (dl *diskLayer) resetCache() {
dl.cleans.Reset()
}
}
// hasher is used to compute the sha256 hash of the provided data.
type hasher struct{ sha crypto.KeccakState }
var hasherPool = sync.Pool{
New: func() interface{} { return &hasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
}
func newHasher() *hasher {
return hasherPool.Get().(*hasher)
}
func (h *hasher) hash(data []byte) common.Hash {
return crypto.HashData(h.sha, data)
}
func (h *hasher) release() {
hasherPool.Put(h)
}

16
triedb/pathdb/errors.go
View file

@ -18,10 +18,6 @@ package pathdb
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
)
var (
@ -45,16 +41,4 @@ var (
// errStateUnrecoverable is returned if state is required to be reverted to
// a destination without associated state history available.
errStateUnrecoverable = errors.New("state is unrecoverable")
// errUnexpectedNode is returned if the requested node with specified path is
// not hash matched with expectation.
errUnexpectedNode = errors.New("unexpected node")
)
func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte, blob []byte) error {
blobHex := "nil"
if len(blob) > 0 {
blobHex = hexutil.Encode(blob)
}
return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x, blob: %s", errUnexpectedNode, loc, owner, path, expHash, gotHash, blobHex)
}

39
triedb/pathdb/journal.go
View file

@ -29,7 +29,6 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/triestate"
)
@ -120,9 +119,10 @@ func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
// loadLayers loads a pre-existing state layer backed by a key-value store.
func (db *Database) loadLayers() layer {
// Retrieve the root node of persistent state.
_, root := rawdb.ReadAccountTrieNode(db.diskdb, nil)
root = types.TrieRootHash(root)
var root = types.EmptyRootHash
if blob := rawdb.ReadAccountTrieNode(db.diskdb, nil); len(blob) > 0 {
root = crypto.Keccak256Hash(blob)
}
// Load the layers by resolving the journal
head, err := db.loadJournal(root)
if err == nil {
@ -162,14 +162,14 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
if err := r.Decode(&encoded); err != nil {
return nil, fmt.Errorf("load disk nodes: %v", err)
}
nodes := make(map[common.Hash]map[string]*trienode.Node)
nodes := make(map[common.Hash]map[string][]byte)
for _, entry := range encoded {
subset := make(map[string]*trienode.Node)
subset := make(map[string][]byte)
for _, n := range entry.Nodes {
if len(n.Blob) > 0 {
subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
subset[string(n.Path)] = n.Blob
} else {
subset[string(n.Path)] = trienode.NewDeleted()
subset[string(n.Path)] = nil
}
}
nodes[entry.Owner] = subset
@ -200,14 +200,14 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
if err := r.Decode(&encoded); err != nil {
return nil, fmt.Errorf("load diff nodes: %v", err)
}
nodes := make(map[common.Hash]map[string]*trienode.Node)
nodes := make(map[common.Hash]map[string][]byte)
for _, entry := range encoded {
subset := make(map[string]*trienode.Node)
subset := make(map[string][]byte)
for _, n := range entry.Nodes {
if len(n.Blob) > 0 {
subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
subset[string(n.Path)] = n.Blob
} else {
subset[string(n.Path)] = trienode.NewDeleted()
subset[string(n.Path)] = nil
}
}
nodes[entry.Owner] = subset
@ -265,7 +265,7 @@ func (dl *diskLayer) journal(w io.Writer) error {
for owner, subset := range dl.buffer.nodes {
entry := journalNodes{Owner: owner}
for path, node := range subset {
entry.Nodes = append(entry.Nodes, journalNode{Path: []byte(path), Blob: node.Blob})
entry.Nodes = append(entry.Nodes, journalNode{Path: []byte(path), Blob: node})
}
nodes = append(nodes, entry)
}
@ -298,7 +298,7 @@ func (dl *diffLayer) journal(w io.Writer) error {
for owner, subset := range dl.nodes {
entry := journalNodes{Owner: owner}
for path, node := range subset {
entry.Nodes = append(entry.Nodes, journalNode{Path: []byte(path), Blob: node.Blob})
entry.Nodes = append(entry.Nodes, journalNode{Path: []byte(path), Blob: node})
}
nodes = append(nodes, entry)
}
@ -361,14 +361,13 @@ func (db *Database) Journal(root common.Hash) error {
if err := rlp.Encode(journal, journalVersion); err != nil {
return err
}
// The stored state in disk might be empty, convert the
// root to emptyRoot in this case.
_, diskroot := rawdb.ReadAccountTrieNode(db.diskdb, nil)
diskroot = types.TrieRootHash(diskroot)
// Secondly write out the state root in disk, ensure all layers
// on top are continuous with disk.
if err := rlp.Encode(journal, diskroot); err != nil {
diskRoot := types.EmptyRootHash
if blob := rawdb.ReadAccountTrieNode(db.diskdb, nil); len(blob) > 0 {
diskRoot = crypto.Keccak256Hash(blob)
}
if err := rlp.Encode(journal, diskRoot); err != nil {
return err
}
// Finally write out the journal of each layer in reverse order.

2
triedb/pathdb/layertree.go
View file

@ -101,7 +101,7 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
if parent == nil {
return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot)
}
l := parent.update(root, parent.stateID()+1, block, nodes.Flatten(), states)
l := parent.update(root, parent.stateID()+1, block, nodes.Slim(), states)
tree.lock.Lock()
tree.layers[l.rootHash()] = l

1
triedb/pathdb/metrics.go
View file

@ -33,6 +33,7 @@ var (
cleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil)
dirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil)
diskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil)
diffFalseMeter = metrics.NewRegisteredMeter("pathdb/diff/false", nil)
commitTimeTimer = metrics.NewRegisteredTimer("pathdb/commit/time", nil)
commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil)

75
triedb/pathdb/nodebuffer.go
View file

@ -17,37 +17,36 @@
package pathdb
import (
"bytes"
"fmt"
"time"
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/trienode"
)
// nodebuffer is a collection of modified trie nodes to aggregate the disk
// write. The content of the nodebuffer must be checked before diving into
// disk (since it basically is not-yet-written data).
type nodebuffer struct {
layers uint64 // The number of diff layers aggregated inside
size uint64 // The size of aggregated writes
limit uint64 // The maximum memory allowance in bytes
nodes map[common.Hash]map[string]*trienode.Node // The dirty node set, mapped by owner and path
layers uint64 // The number of diff layers aggregated inside
size uint64 // The size of aggregated writes
limit uint64 // The maximum memory allowance in bytes
nodes map[common.Hash]map[string][]byte // The aggregated node set, mapped by owner and path
}
// newNodeBuffer initializes the node buffer with the provided nodes.
func newNodeBuffer(limit int, nodes map[common.Hash]map[string]*trienode.Node, layers uint64) *nodebuffer {
func newNodeBuffer(limit int, nodes map[common.Hash]map[string][]byte, layers uint64) *nodebuffer {
if nodes == nil {
nodes = make(map[common.Hash]map[string]*trienode.Node)
nodes = make(map[common.Hash]map[string][]byte)
}
var size uint64
for _, subset := range nodes {
for path, n := range subset {
size += uint64(len(n.Blob) + len(path))
size += uint64(len(n) + len(path))
}
}
return &nodebuffer{
@ -59,28 +58,22 @@ func newNodeBuffer(limit int, nodes map[common.Hash]map[string]*trienode.Node, l
}
// node retrieves the trie node with given node info.
func (b *nodebuffer) node(owner common.Hash, path []byte, hash common.Hash) (*trienode.Node, error) {
func (b *nodebuffer) node(owner common.Hash, path []byte) ([]byte, bool) {
subset, ok := b.nodes[owner]
if !ok {
return nil, nil
return nil, false
}
n, ok := subset[string(path)]
if !ok {
return nil, nil
return nil, false
}
if n.Hash != hash {
dirtyFalseMeter.Mark(1)
log.Error("Unexpected trie node in node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
}
return n, nil
return n, true
}
// commit merges the dirty nodes into the nodebuffer. This operation won't take
// the ownership of the nodes map which belongs to the bottom-most diff layer.
// It will just hold the node references from the given map which are safe to
// copy.
func (b *nodebuffer) commit(nodes map[common.Hash]map[string]*trienode.Node) *nodebuffer {
// It will hold the node references from the given map which are safe to copy.
func (b *nodebuffer) commit(nodes map[common.Hash]map[string][]byte) *nodebuffer {
var (
delta int64
overwrite int64
@ -94,22 +87,25 @@ func (b *nodebuffer) commit(nodes map[common.Hash]map[string]*trienode.Node) *no
// The nodes belong to original diff layer are still accessible even
// after merging, thus the ownership of nodes map should still belong
// to original layer and any mutation on it should be prevented.
current = make(map[string]*trienode.Node)
current = make(map[string][]byte)
for path, n := range subset {
// node deep-copying is not worthwhile as they are immutable.
current[path] = n
delta += int64(len(n.Blob) + len(path))
delta += int64(len(n) + len(path))
}
b.nodes[owner] = current
continue
}
// Merge the node sets belonging to the same owner.
for path, n := range subset {
if orig, exist := current[path]; !exist {
delta += int64(len(n.Blob) + len(path))
delta += int64(len(n) + len(path))
} else {
delta += int64(len(n.Blob) - len(orig.Blob))
delta += int64(len(n) - len(orig))
overwrite++
overwriteSize += int64(len(orig.Blob) + len(path))
overwriteSize += int64(len(orig) + len(path))
}
// node deep-copying is not worthwhile as they are immutable.
current[path] = n
}
b.nodes[owner] = current
@ -124,7 +120,7 @@ func (b *nodebuffer) commit(nodes map[common.Hash]map[string]*trienode.Node) *no
// revert is the reverse operation of commit. It also merges the provided nodes
// into the nodebuffer, the difference is that the provided node set should
// revert the changes made by the last state transition.
func (b *nodebuffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) error {
func (b *nodebuffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string][]byte) error {
// Short circuit if no embedded state transition to revert.
if b.layers == 0 {
return errStateUnrecoverable
@ -153,20 +149,21 @@ func (b *nodebuffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[s
//
// In case of database rollback, don't panic if this "clean"
// node occurs which is not present in buffer.
var nhash common.Hash
var blob []byte
if owner == (common.Hash{}) {
_, nhash = rawdb.ReadAccountTrieNode(db, []byte(path))
blob = rawdb.ReadAccountTrieNode(db, []byte(path))
} else {
_, nhash = rawdb.ReadStorageTrieNode(db, owner, []byte(path))
blob = rawdb.ReadStorageTrieNode(db, owner, []byte(path))
}
// Ignore the clean node in the case described above.
if nhash == n.Hash {
if bytes.Equal(blob, n) {
continue
}
panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex()))
panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, n))
}
// node deep-copying is not worthwhile as they are immutable.
current[path] = n
delta += int64(len(n.Blob)) - int64(len(orig.Blob))
delta += int64(len(n)) - int64(len(orig))
}
}
b.updateSize(delta)
@ -189,7 +186,7 @@ func (b *nodebuffer) updateSize(delta int64) {
func (b *nodebuffer) reset() {
b.layers = 0
b.size = 0
b.nodes = make(map[common.Hash]map[string]*trienode.Node)
b.nodes = make(map[common.Hash]map[string][]byte)
}
// empty returns an indicator if nodebuffer contains any state transition inside.
@ -251,10 +248,10 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui
// writeNodes writes the trie nodes into the provided database batch.
// Note this function will also inject all the newly written nodes
// into clean cache.
func writeNodes(batch ethdb.Batch, nodes map[common.Hash]map[string]*trienode.Node, clean *fastcache.Cache) (total int) {
func writeNodes(batch ethdb.Batch, nodes map[common.Hash]map[string][]byte, clean *fastcache.Cache) (total int) {
for owner, subset := range nodes {
for path, n := range subset {
if n.IsDeleted() {
if len(n) == 0 {
if owner == (common.Hash{}) {
rawdb.DeleteAccountTrieNode(batch, []byte(path))
} else {
@ -265,12 +262,12 @@ func writeNodes(batch ethdb.Batch, nodes map[common.Hash]map[string]*trienode.No
}
} else {
if owner == (common.Hash{}) {
rawdb.WriteAccountTrieNode(batch, []byte(path), n.Blob)
rawdb.WriteAccountTrieNode(batch, []byte(path), n)
} else {
rawdb.WriteStorageTrieNode(batch, owner, []byte(path), n.Blob)
rawdb.WriteStorageTrieNode(batch, owner, []byte(path), n)
}
if clean != nil {
clean.Set(cacheKey(owner, []byte(path)), n.Blob)
clean.Set(cacheKey(owner, []byte(path)), n)
}
}
}

89
triedb/pathdb/reader.go Normal file
View file

@ -0,0 +1,89 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
package pathdb
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/triedb/database"
)
// The types of locations where the node is found.
const (
locDirtyCache = "dirty" // dirty cache
locCleanCache = "clean" // clean cache
locDiskLayer = "disk" // persistent state
locDiffLayer = "diff" // diff layers
)
// nodeLoc is a helpful structure that contains the location where the node
// is found, as it's useful for debugging purposes.
type nodeLoc struct {
loc string
depth int
}
// string returns the string representation of node location.
func (loc *nodeLoc) string() string {
return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth)
}
// reader implements the Reader interface, providing the functionalities to
// retrieve trie nodes by wrapping the internal state layer.
type reader struct {
layer layer
state crypto.KeccakState
}
// Node implements trie.Reader interface, retrieving the node with specified
// node info. Don't modify the returned byte slice since it's not deep-copied
// and still be referenced by database.
func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
blob, loc, err := r.layer.node(owner, path, 0)
if err != nil {
return nil, err
}
if got := crypto.HashData(r.state, blob); got != hash {
// Location is always available even if the node
// is not found.
switch loc.loc {
case locCleanCache:
cleanFalseMeter.Mark(1)
case locDirtyCache:
dirtyFalseMeter.Mark(1)
case locDiffLayer:
diffFalseMeter.Mark(1)
case locDiskLayer:
diskFalseMeter.Mark(1)
}
log.Error("Unexpected trie node", "location", loc.loc, "owner", owner, "path", path, "expect", hash, "got", got)
return nil, fmt.Errorf("unexpected node: (%x %v), %x!=%x, %s", owner, path, hash, got, loc.string())
}
return blob, nil
}
// Reader retrieves a layer belonging to the given state root.
func (db *Database) Reader(root common.Hash) (database.Reader, error) {
layer := db.tree.get(root)
if layer == nil {
return nil, fmt.Errorf("state %#x is not available", root)
}
return &reader{layer: layer, state: crypto.NewKeccakState()}, nil
}