forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-31 09:03:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'ethereum:master' into portal

Browse source
This commit is contained in:
Chen Kai 2024-07-17 21:45:57 +08:00 committed by GitHub
commit 3816cc97e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 112 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/geth/dbcmd.go
View file

@ -248,7 +248,8 @@ func removeDB(ctx *cli.Context) error {
// Delete state data
statePaths := []string{
rootDir,
filepath.Join(ancientDir, rawdb.StateFreezerName),
filepath.Join(ancientDir, rawdb.MerkleStateFreezerName),
filepath.Join(ancientDir, rawdb.VerkleStateFreezerName),
}
confirmAndRemoveDB(statePaths, "state data", ctx, removeStateDataFlag.Name)

6
core/genesis_test.go
View file

@ -311,7 +311,7 @@ func TestVerkleGenesisCommit(t *testing.T) {
}
db := rawdb.NewMemoryDatabase()
triedb := triedb.NewDatabase(db, &triedb.Config{IsVerkle: true, PathDB: pathdb.Defaults})
triedb := triedb.NewDatabase(db, triedb.VerkleDefaults)
block := genesis.MustCommit(db, triedb)
if !bytes.Equal(block.Root().Bytes(), expected) {
t.Fatalf("invalid genesis state root, expected %x, got %x", expected, block.Root())
@ -321,8 +321,8 @@ func TestVerkleGenesisCommit(t *testing.T) {
if !triedb.IsVerkle() {
t.Fatalf("expected trie to be verkle")
}
if !rawdb.HasAccountTrieNode(db, nil) {
vdb := rawdb.NewTable(db, string(rawdb.VerklePrefix))
if !rawdb.HasAccountTrieNode(vdb, nil) {
t.Fatal("could not find node")
}
}

12
core/rawdb/accessors_trie.go
View file

@ -245,7 +245,7 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
func ReadStateScheme(db ethdb.Database) string {
// Check if state in path-based scheme is present.
if HasAccountTrieNode(db, nil) {
return PathScheme
@ -255,6 +255,16 @@ func ReadStateScheme(db ethdb.Reader) string {
if id := ReadPersistentStateID(db); id != 0 {
return PathScheme
}
// Check if verkle state in path-based scheme is present.
vdb := NewTable(db, string(VerklePrefix))
if HasAccountTrieNode(vdb, nil) {
return PathScheme
}
// The root node of verkle might be deleted during the initial snap sync,
// check the persistent state id then.
if id := ReadPersistentStateID(vdb); id != 0 {
return PathScheme
}
// In a hash-based scheme, the genesis state is consistently stored
// on the disk. To assess the scheme of the persistent state, it
// suffices to inspect the scheme of the genesis state.

17
core/rawdb/ancient_scheme.go
View file

@ -72,12 +72,13 @@ var stateFreezerNoSnappy = map[string]bool{
// The list of identifiers of ancient stores.
var (
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
StateFreezerName = "state" // the folder name of reverse diff ancient store.
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
)
// freezers the collections of all builtin freezers.
var freezers = []string{ChainFreezerName, StateFreezerName}
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName}
// NewStateFreezer initializes the ancient store for state history.
//
@ -85,9 +86,15 @@ var freezers = []string{ChainFreezerName, StateFreezerName}
// state freezer (e.g. dev mode).
// - if non-empty directory is given, initializes the regular file-based
// state freezer.
func NewStateFreezer(ancientDir string, readOnly bool) (ethdb.ResettableAncientStore, error) {
func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
if ancientDir == "" {
return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil
}
return newResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
var name string
if verkle {
name = filepath.Join(ancientDir, VerkleStateFreezerName)
} else {
name = filepath.Join(ancientDir, MerkleStateFreezerName)
}
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
}

6
core/rawdb/ancient_utils.go
View file

@ -88,12 +88,12 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
}
infos = append(infos, info)
case StateFreezerName:
case MerkleStateFreezerName, VerkleStateFreezerName:
datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
f, err := NewStateFreezer(datadir, freezer == VerkleStateFreezerName, true)
if err != nil {
continue // might be possible the state freezer is not existent
}
@ -124,7 +124,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s
switch freezerName {
case ChainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
case StateFreezerName:
case MerkleStateFreezerName, VerkleStateFreezerName:
path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)

24
core/rawdb/database.go
View file

@ -481,6 +481,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
beaconHeaders stat
cliqueSnaps stat
// Verkle statistics
verkleTries stat
verkleStateLookups stat
// Les statistic
chtTrieNodes stat
bloomTrieNodes stat
@ -550,6 +554,24 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
bloomTrieNodes.Add(size)
// Verkle trie data is detected, determine the sub-category
case bytes.HasPrefix(key, VerklePrefix):
remain := key[len(VerklePrefix):]
switch {
case IsAccountTrieNode(remain):
verkleTries.Add(size)
case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength:
verkleStateLookups.Add(size)
case bytes.Equal(remain, persistentStateIDKey):
metadata.Add(size)
case bytes.Equal(remain, trieJournalKey):
metadata.Add(size)
case bytes.Equal(remain, snapSyncStatusFlagKey):
metadata.Add(size)
default:
unaccounted.Add(size)
}
default:
var accounted bool
for _, meta := range [][]byte{
@ -590,6 +612,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()},
{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

7
core/rawdb/schema.go
View file

@ -117,6 +117,13 @@ var (
TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node
stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id
// VerklePrefix is the database prefix for Verkle trie data, which includes:
// (a) Trie nodes
// (b) In-memory trie node journal
// (c) Persistent state ID
// (d) State ID lookups, etc.
VerklePrefix = []byte("v")
PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db

14
core/state/statedb.go
View file

@ -860,12 +860,16 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
}
obj := s.stateObjects[addr] // closure for the task runner below
workers.Go(func() error {
obj.updateRoot()
if s.db.TrieDB().IsVerkle() {
obj.updateTrie()
} else {
obj.updateRoot()
// If witness building is enabled and the state object has a trie,
// gather the witnesses for its specific storage trie
if s.witness != nil && obj.trie != nil {
s.witness.AddState(obj.trie.Witness())
// If witness building is enabled and the state object has a trie,
// gather the witnesses for its specific storage trie
if s.witness != nil && obj.trie != nil {
s.witness.AddState(obj.trie.Witness())
}
}
return nil
})

15
core/txpool/legacypool/legacypool.go
View file

@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/holiman/uint256"
"golang.org/x/exp/maps"
)
const (
@ -1717,7 +1718,7 @@ func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
type accountSet struct {
accounts map[common.Address]struct{}
signer types.Signer
cache *[]common.Address
cache []common.Address
}
// newAccountSet creates a new address set with an associated signer for sender
@ -1765,20 +1766,14 @@ func (as *accountSet) addTx(tx *types.Transaction) {
// reuse. The returned slice should not be changed!
func (as *accountSet) flatten() []common.Address {
if as.cache == nil {
accounts := make([]common.Address, 0, len(as.accounts))
for account := range as.accounts {
accounts = append(accounts, account)
}
as.cache = &accounts
as.cache = maps.Keys(as.accounts)
}
return *as.cache
return as.cache
}
// merge adds all addresses from the 'other' set into 'as'.
func (as *accountSet) merge(other *accountSet) {
for addr := range other.accounts {
as.accounts[addr] = struct{}{}
}
maps.Copy(as.accounts, other.accounts)
as.cache = nil
}

8
trie/committer.go
View file

@ -154,12 +154,8 @@ func (c *committer) store(path []byte, n node) node {
return hash
}
// MerkleResolver the children resolver in merkle-patricia-tree.
type MerkleResolver struct{}
// ForEach implements childResolver, decodes the provided node and
// traverses the children inside.
func (resolver MerkleResolver) ForEach(node []byte, onChild func(common.Hash)) {
// ForGatherChildren decodes the provided node and traverses the children inside.
func ForGatherChildren(node []byte, onChild func(common.Hash)) {
forGatherChildren(mustDecodeNodeUnsafe(nil, node), onChild)
}

21
triedb/database.go
View file

@ -23,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/triestate"
"github.com/ethereum/go-ethereum/triedb/database"
@ -43,9 +42,18 @@ type Config struct {
// default settings.
var HashDefaults = &Config{
Preimages: false,
IsVerkle: false,
HashDB: hashdb.Defaults,
}
// VerkleDefaults represents a config for holding verkle trie data
// using path-based scheme with default settings.
var VerkleDefaults = &Config{
Preimages: false,
IsVerkle: true,
PathDB: pathdb.Defaults,
}
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {
@ -85,7 +93,6 @@ type backend interface {
// relevant with trie nodes and node preimages.
type Database struct {
config *Config // Configuration for trie database
diskdb ethdb.Database // Persistent database to store the snapshot
preimages *preimageStore // The store for caching preimages
backend backend // The backend for managing trie nodes
}
@ -103,7 +110,6 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
}
db := &Database{
config: config,
diskdb: diskdb,
preimages: preimages,
}
if config.HashDB != nil && config.PathDB != nil {
@ -112,14 +118,7 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
if config.PathDB != nil {
db.backend = pathdb.New(diskdb, config.PathDB, config.IsVerkle)
} else {
var resolver hashdb.ChildResolver
if config.IsVerkle {
// TODO define verkle resolver
log.Crit("verkle does not use a hash db")
} else {
resolver = trie.MerkleResolver{}
}
db.backend = hashdb.New(diskdb, config.HashDB, resolver)
db.backend = hashdb.New(diskdb, config.HashDB)
}
return db
}

30
triedb/hashdb/database.go
View file

@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/triestate"
"github.com/ethereum/go-ethereum/triedb/database"
@ -60,12 +61,6 @@ var (
memcacheCommitBytesMeter = metrics.NewRegisteredMeter("hashdb/memcache/commit/bytes", nil)
)
// ChildResolver defines the required method to decode the provided
// trie node and iterate the children on top.
type ChildResolver interface {
ForEach(node []byte, onChild func(common.Hash))
}
// Config contains the settings for database.
type Config struct {
CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes
@ -84,9 +79,7 @@ var Defaults = &Config{
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
type Database struct {
diskdb ethdb.Database // Persistent storage for matured trie nodes
resolver ChildResolver // The handler to resolve children of nodes
diskdb ethdb.Database // Persistent storage for matured trie nodes
cleans *fastcache.Cache // GC friendly memory cache of clean node RLPs
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty trie nodes
oldest common.Hash // Oldest tracked node, flush-list head
@ -124,15 +117,15 @@ var cachedNodeSize = int(reflect.TypeOf(cachedNode{}).Size())
// forChildren invokes the callback for all the tracked children of this node,
// both the implicit ones from inside the node as well as the explicit ones
// from outside the node.
func (n *cachedNode) forChildren(resolver ChildResolver, onChild func(hash common.Hash)) {
func (n *cachedNode) forChildren(onChild func(hash common.Hash)) {
for child := range n.external {
onChild(child)
}
resolver.ForEach(n.node, onChild)
trie.ForGatherChildren(n.node, onChild)
}
// New initializes the hash-based node database.
func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Database {
func New(diskdb ethdb.Database, config *Config) *Database {
if config == nil {
config = Defaults
}
@ -141,10 +134,9 @@ func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Databas
cleans = fastcache.New(config.CleanCacheSize)
}
return &Database{
diskdb: diskdb,
resolver: resolver,
cleans: cleans,
dirties: make(map[common.Hash]*cachedNode),
diskdb: diskdb,
cleans: cleans,
dirties: make(map[common.Hash]*cachedNode),
}
}
@ -163,7 +155,7 @@ func (db *Database) insert(hash common.Hash, node []byte) {
node: node,
flushPrev: db.newest,
}
entry.forChildren(db.resolver, func(child common.Hash) {
entry.forChildren(func(child common.Hash) {
if c := db.dirties[child]; c != nil {
c.parents++
}
@ -316,7 +308,7 @@ func (db *Database) dereference(hash common.Hash) {
db.dirties[node.flushNext].flushPrev = node.flushPrev
}
// Dereference all children and delete the node
node.forChildren(db.resolver, func(child common.Hash) {
node.forChildren(func(child common.Hash) {
db.dereference(child)
})
delete(db.dirties, hash)
@ -465,7 +457,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane
var err error
// Dereference all children and delete the node
node.forChildren(db.resolver, func(child common.Hash) {
node.forChildren(func(child common.Hash) {
if err == nil {
err = db.commit(child, batch, uncacher)
}

10
triedb/pathdb/database.go
View file

@ -152,6 +152,14 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
}
config = config.sanitize()
// Establish a dedicated database namespace tailored for verkle-specific
// data, ensuring the isolation of both verkle and merkle tree data. It's
// important to note that the introduction of a prefix won't lead to
// substantial storage overhead, as the underlying database will efficiently
// compress the shared key prefix.
if isVerkle {
diskdb = rawdb.NewTable(diskdb, string(rawdb.VerklePrefix))
}
db := &Database{
readOnly: config.ReadOnly,
isVerkle: isVerkle,
@ -190,7 +198,7 @@ func (db *Database) repairHistory() error {
// all of them. Fix the tests first.
return nil
}
freezer, err := rawdb.NewStateFreezer(ancient, db.readOnly)
freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly)
if err != nil {
log.Crit("Failed to open state history freezer", "err", err)
}

8
triedb/pathdb/history_test.go
View file

@ -129,7 +129,7 @@ func TestTruncateHeadHistory(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -157,7 +157,7 @@ func TestTruncateTailHistory(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -200,7 +200,7 @@ func TestTruncateTailHistories(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false, false)
)
defer freezer.Close()
@ -228,7 +228,7 @@ func TestTruncateOutOfRange(t *testing.T) {
var (
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()