save current state after discussion with Gary

This commit is contained in:
Guillaume Ballet 2025-04-03 20:45:00 +02:00
parent 8e8d10ac6c
commit 7942e5211c
19 changed files with 190 additions and 254 deletions

View file

@ -416,7 +416,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB {
tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true})
sdb := state.NewDatabase(tdb, nil)
vdb := triedb.NewDatabase(db, nil)
sdb := state.NewDatabase(tdb, vdb, nil)
statedb, _ := state.New(types.EmptyRootHash, sdb)
for addr, a := range accounts {
statedb.SetCode(addr, a.Code)

View file

@ -221,13 +221,14 @@ func runCmd(ctx *cli.Context) error {
}
db := rawdb.NewMemoryDatabase()
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
triedb := triedb.NewDatabase(db, &triedb.Config{
Preimages: preimages,
HashDB: hashdb.Defaults,
})
defer triedb.Close()
genesis := genesisConfig.MustCommit(db, triedb)
sdb := state.NewDatabase(triedb, nil)
genesis := genesisConfig.MustCommit(db, triedb, verkledb)
sdb := state.NewDatabase(triedb, verkledb, nil)
prestate, _ = state.New(genesis.Root(), sdb)
chainConfig = genesisConfig.Config

View file

@ -244,10 +244,11 @@ func initGenesis(ctx *cli.Context) error {
}
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
triedb, verkledb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
defer triedb.Close()
defer verkledb.Close()
_, hash, _, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
_, hash, _, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, verkledb, genesis, &overrides)
if err != nil {
utils.Fatalf("Failed to write genesis block: %v", err)
}
@ -591,10 +592,11 @@ func dump(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup
triedb, verkledb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup
defer triedb.Close()
defer verkledb.Close()
state, err := state.New(root, state.NewDatabase(triedb, nil))
state, err := state.New(root, state.NewDatabase(triedb, verkledb, nil))
if err != nil {
return err
}

View file

@ -546,8 +546,9 @@ func dbDumpTrie(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
defer verkledb.Close()
var (
state []byte
@ -881,8 +882,9 @@ func inspectHistory(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
triedb := utils.MakeTrieDatabase(ctx, db, false, false, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, db, false, false)
defer triedb.Close()
defer verkledb.Close()
var (
err error

View file

@ -217,7 +217,8 @@ func verifyState(ctx *cli.Context) error {
log.Error("Failed to load head block")
return errors.New("no head block")
}
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer verkledb.Close()
defer triedb.Close()
snapConfig := snapshot.Config{
@ -272,8 +273,9 @@ func traverseState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
defer verkledb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
@ -381,8 +383,9 @@ func traverseRawState(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
defer verkledb.Close()
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
@ -548,8 +551,9 @@ func dumpState(ctx *cli.Context) error {
if err != nil {
return err
}
triedb := utils.MakeTrieDatabase(ctx, db, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, db, false, true)
defer triedb.Close()
defer verkledb.Close()
snapConfig := snapshot.Config{
CacheSize: 256,
@ -630,8 +634,9 @@ func snapshotExportPreimages(ctx *cli.Context) error {
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false)
triedb, verkledb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
defer triedb.Close()
defer verkledb.Close()
var root common.Hash
if ctx.NArg() > 1 {

View file

@ -2240,10 +2240,9 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
}
// MakeTrieDatabase constructs a trie database based on the configured scheme.
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *triedb.Database {
func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) (*triedb.Database, *triedb.Database) {
config := &triedb.Config{
Preimages: preimage,
IsVerkle: isVerkle,
}
scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk)
if err != nil {
@ -2254,12 +2253,12 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read
// ignore the parameter silently. TODO(rjl493456442)
// please config it if read mode is implemented.
config.HashDB = hashdb.Defaults
return triedb.NewDatabase(disk, config)
return triedb.NewDatabase(disk, config), triedb.NewDatabase(disk, triedb.VerkleDefaults)
}
if readOnly {
config.PathDB = pathdb.ReadOnly
} else {
config.PathDB = pathdb.Defaults
}
return triedb.NewDatabase(disk, config)
return triedb.NewDatabase(disk, config), triedb.NewDatabase(disk, triedb.VerkleDefaults)
}

View file

@ -161,10 +161,9 @@ type CacheConfig struct {
}
// triedbConfig derives the configures for trie database.
func (c *CacheConfig) triedbConfig(isVerkle bool) *triedb.Config {
func (c *CacheConfig) triedbConfig() *triedb.Config {
config := &triedb.Config{
Preimages: c.Preimages,
IsVerkle: isVerkle,
}
if c.StateScheme == rawdb.HashScheme {
config.HashDB = &hashdb.Config{
@ -232,6 +231,7 @@ type BlockChain struct {
lastWrite uint64 // Last block when the state was flushed
flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state
triedb *triedb.Database // The database handler for maintaining trie nodes.
verkledb *triedb.Database // The database handler for maintaining verkle nodes.
statedb *state.CachingDB // State database to reuse between imports (contains state cache)
txIndexer *txIndexer // Transaction indexer, might be nil if not enabled
@ -283,17 +283,15 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig = defaultCacheConfig
}
// Open trie database with provided config
enableVerkle, err := EnableVerkleAtGenesis(db, genesis)
if err != nil {
return nil, err
}
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(enableVerkle))
// TODO(gballet) remove EnableVerkleAtGenesis if it's no longer used
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig())
// Write the supplied genesis to the database if it has not been initialized
// yet. The corresponding chain config will be returned, either from the
// provided genesis or from the locally stored configuration if the genesis
// has already been initialized.
chainConfig, genesisHash, compatErr, err := SetupGenesisBlockWithOverride(db, triedb, genesis, overrides)
chainConfig, genesisHash, compatErr, err := SetupGenesisBlockWithOverride(db, triedb, verkledb, genesis, overrides)
if err != nil {
return nil, err
}
@ -310,6 +308,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig: cacheConfig,
db: db,
triedb: triedb,
verkledb: verkledb,
triegc: prque.New[int64, common.Hash](nil),
quit: make(chan struct{}),
chainmu: syncx.NewClosableMutex(),
@ -327,7 +326,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
return nil, err
}
bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit))
bc.statedb = state.NewDatabase(bc.triedb, nil)
bc.statedb = state.NewDatabase(bc.triedb, bc.verkledb, nil)
bc.validator = NewBlockValidator(chainConfig, bc)
bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc)
bc.processor = NewStateProcessor(chainConfig, bc.hc)
@ -468,7 +467,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root)
// Re-initialize the state database with snapshot
bc.statedb = state.NewDatabase(bc.triedb, bc.snaps)
bc.statedb = state.NewDatabase(bc.triedb, bc.verkledb, bc.snaps)
}
// Rewind the chain in case of an incompatible config upgrade.

View file

@ -422,12 +422,15 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
return block, b.receipts
}
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
defer verkledb.Close()
// Forcibly use hash-based state scheme for retaining all nodes in disk.
triedb := triedb.NewDatabase(db, triedb.HashDefaults)
defer triedb.Close()
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(triedb, nil))
statedb, err := state.New(parent.Root(), state.NewDatabase(triedb, verkledb, nil))
if err != nil {
panic(err)
}
@ -468,9 +471,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
// then generate chain on top.
func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) {
db := rawdb.NewMemoryDatabase()
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
defer verkledb.Close()
triedb := triedb.NewDatabase(db, triedb.HashDefaults)
defer triedb.Close()
_, err := genesis.Commit(db, triedb)
_, err := genesis.Commit(db, triedb, verkledb)
if err != nil {
panic(err)
}
@ -478,7 +483,7 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int,
return db, blocks, receipts
}
func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb, vktdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
if config == nil {
config = params.TestChainConfig
}
@ -537,7 +542,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabase(trdb, nil))
statedb, err := state.New(parent.Root(), state.NewDatabase(trdb, vktdb, nil))
if err != nil {
panic(err)
}
@ -577,13 +582,15 @@ func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n
db := rawdb.NewMemoryDatabase()
cacheConfig := DefaultCacheConfigWithScheme(rawdb.PathScheme)
cacheConfig.SnapshotLimit = 0
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true))
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
defer verkledb.Close()
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig())
defer triedb.Close()
genesisBlock, err := genesis.Commit(db, triedb)
genesisBlock, err := genesis.Commit(db, triedb, verkledb)
if err != nil {
panic(err)
}
blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen)
blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, verkledb, n, gen)
return genesisBlock.Hash(), db, blocks, receipts, proofs, keyvals
}

View file

@ -18,6 +18,7 @@ package core
import (
"bytes"
"encoding/gob"
"encoding/json"
"errors"
"fmt"
@ -145,7 +146,7 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) {
emptyRoot = types.EmptyVerkleHash
}
db := rawdb.NewMemoryDatabase()
statedb, err := state.New(emptyRoot, state.NewDatabase(triedb.NewDatabase(db, config), nil))
statedb, err := state.New(emptyRoot, state.NewDatabase(triedb.NewDatabase(db, config), triedb.NewDatabase(db, triedb.VerkleDefaults), nil))
if err != nil {
return common.Hash{}, err
}
@ -164,12 +165,12 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) {
// flushAlloc is very similar with hash, but the main difference is all the
// generated states will be persisted into the given database.
func flushAlloc(ga *types.GenesisAlloc, triedb *triedb.Database) (common.Hash, error) {
func flushAlloc(ga *types.GenesisAlloc, triedb, verkledb *triedb.Database) (common.Hash, error) {
emptyRoot := types.EmptyRootHash
if triedb.IsVerkle() {
emptyRoot = types.EmptyVerkleHash
}
statedb, err := state.New(emptyRoot, state.NewDatabase(triedb, nil))
statedb, err := state.New(emptyRoot, state.NewDatabase(triedb, verkledb, nil))
if err != nil {
return common.Hash{}, err
}
@ -276,6 +277,20 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error {
return cfg.CheckConfigForkOrder()
}
// saveVerkleTransitionStatusAtVerlkeGenesis saves a conversion marker
// representing a converted state, which is used in devnets that activate
// verkle at genesis.
func saveVerkleTransitionStatusAtVerlkeGenesis(db ethdb.Database) {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err := enc.Encode(&state.TransitionState{Ended: true})
if err != nil {
log.Error("failed to encode transition state", "err", err)
return
}
rawdb.WriteVerkleTransitionState(db, common.Hash{}, buf.Bytes())
}
// SetupGenesisBlock writes or updates the genesis block in db.
// The block that will be used is:
//
@ -287,11 +302,11 @@ func (o *ChainOverrides) apply(cfg *params.ChainConfig) error {
// The stored chain configuration will be updated if it is compatible (i.e. does not
// specify a fork block below the local head block). In case of a conflict, the
// error is a *params.ConfigCompatError and the new, unwritten config is returned.
func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
return SetupGenesisBlockWithOverride(db, triedb, genesis, nil)
func SetupGenesisBlock(db ethdb.Database, triedb, verkledb *triedb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
return SetupGenesisBlockWithOverride(db, triedb, verkledb, genesis, nil)
}
func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
func SetupGenesisBlockWithOverride(db ethdb.Database, triedb, verkledb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
// Copy the genesis, so we can operate on a copy.
genesis = genesis.copy()
// Sanitize the supplied genesis, ensuring it has the associated chain
@ -299,6 +314,13 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
if genesis != nil && genesis.Config == nil {
return nil, common.Hash{}, nil, errGenesisNoConfig
}
// In case of verkle-at-genesis, we need to ensure that the conversion
// markers are indicating that the conversion has completed.
if genesis != nil && genesis.Config.VerkleTime != nil && *genesis.Config.VerkleTime == genesis.Timestamp {
saveVerkleTransitionStatusAtVerlkeGenesis(db)
}
// Commit the genesis if the database is empty
ghash := rawdb.ReadCanonicalHash(db, 0)
if (ghash == common.Hash{}) {
@ -312,7 +334,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
return nil, common.Hash{}, nil, err
}
block, err := genesis.Commit(db, triedb)
block, err := genesis.Commit(db, triedb, verkledb)
if err != nil {
return nil, common.Hash{}, nil, err
}
@ -340,7 +362,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
if hash := genesis.ToBlock().Hash(); hash != ghash {
return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash}
}
block, err := genesis.Commit(db, triedb)
block, err := genesis.Commit(db, triedb, verkledb)
if err != nil {
return nil, common.Hash{}, nil, err
}
@ -521,7 +543,7 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block {
// Commit writes the block and state of a genesis specification to the database.
// The block is committed as the canonical head block.
func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) {
func (g *Genesis) Commit(db ethdb.Database, triedb, verkledb *triedb.Database) (*types.Block, error) {
if g.Number != 0 {
return nil, errors.New("can't commit genesis block with number > 0")
}
@ -536,7 +558,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
return nil, errors.New("can't start clique chain without signers")
}
// flush the data to disk and compute the state root
root, err := flushAlloc(&g.Alloc, triedb)
root, err := flushAlloc(&g.Alloc, triedb, verkledb)
if err != nil {
return nil, err
}
@ -561,8 +583,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
// MustCommit writes the genesis block and state to db, panicking on error.
// The block is committed as the canonical head block.
func (g *Genesis) MustCommit(db ethdb.Database, triedb *triedb.Database) *types.Block {
block, err := g.Commit(db, triedb)
func (g *Genesis) MustCommit(db ethdb.Database, triedb, verkledb *triedb.Database) *types.Block {
block, err := g.Commit(db, triedb, verkledb)
if err != nil {
panic(err)
}

View file

@ -64,14 +64,14 @@ func testSetupGenesis(t *testing.T, scheme string) {
{
name: "genesis without ChainConfig",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), new(Genesis))
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), triedb.NewDatabase(db, newDbConfig(scheme)), new(Genesis))
},
wantErr: errGenesisNoConfig,
},
{
name: "no block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil)
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), triedb.NewDatabase(db, newDbConfig(scheme)), nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,
@ -80,7 +80,7 @@ func testSetupGenesis(t *testing.T, scheme string) {
name: "mainnet block in DB, genesis == nil",
fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) {
DefaultGenesisBlock().MustCommit(db, triedb.NewDatabase(db, newDbConfig(scheme)))
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), nil)
return SetupGenesisBlock(db, triedb.NewDatabase(db, newDbConfig(scheme)), triedb.NewDatabase(db, newDbConfig(scheme)), nil)
},
wantHash: params.MainnetGenesisHash,
wantConfig: params.MainnetChainConfig,

View file

@ -216,19 +216,18 @@ func (kvm *keyValueMigrator) migrateCollectedKeyValues(tree *trie.VerkleTrie) er
// OverlayVerkleTransition contains the overlay conversion logic
func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedCount uint64) error {
migrdb := statedb.Database()
migrdb.LockCurrentTransitionState()
defer migrdb.UnLockCurrentTransitionState()
ts := migrdb.LoadTransitionState(root)
// verkle transition: if the conversion process is in progress, move
// N values from the MPT into the verkle tree.
if migrdb.InTransition() {
log.Debug("Processing verkle conversion starting", "account hash", migrdb.GetCurrentAccountHash(), "slot hash", migrdb.GetCurrentSlotHash(), "state root", root)
if ts.InTransition() {
log.Debug("Processing verkle conversion starting", "account hash", ts.GetCurrentAccountHash(), "slot hash", ts.CurrentSlotHash, "state root", root)
var (
now = time.Now()
tt = statedb.GetTrie().(*trie.TransitionTrie)
mpt = tt.Base()
hasPreimagesBin = false
preimageSeek = migrdb.GetCurrentPreimageOffset()
preimageSeek = ts.CurrentPreimageOffset
fpreimages *bufio.Reader
)
@ -246,7 +245,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
hasPreimagesBin = true
}
accIt, err := statedb.Database().Snapshot().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash())
accIt, err := statedb.Database().Snapshot().AccountIterator(mpt.Hash(), ts.GetCurrentAccountHash())
if err != nil {
return err
}
@ -254,7 +253,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
accIt.Next()
// If we're about to start with the migration process, we have to read the first account hash preimage.
if migrdb.GetCurrentAccountAddress() == nil {
if ts.CurrentAccountAddress == nil {
var addr common.Address
if hasPreimagesBin {
if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
@ -266,8 +265,8 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
}
}
migrdb.SetCurrentAccountAddress(addr)
if migrdb.GetCurrentAccountHash() != accIt.Hash() {
ts.CurrentAccountAddress = &addr
if ts.GetCurrentAccountHash() != accIt.Hash() {
return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
}
preimageSeek += int64(len(addr))
@ -299,7 +298,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
// to during normal block execution. A mitigation strategy has been
// introduced with the `*StorageRootConversion` fields in VerkleDB.
if len(acc.Root) == 32 && acc.Root != types.EmptyRootHash {
stIt, err := statedb.Database().Snapshot().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash())
stIt, err := statedb.Database().Snapshot().StorageIterator(mpt.Hash(), accIt.Hash(), ts.CurrentSlotHash)
if err != nil {
return err
}
@ -316,8 +315,8 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
// processing the storage for that account where we left off.
// If the entire storage was processed, then the iterator was
// created in vain, but it's ok as this will not happen often.
for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ {
log.Trace("Processing storage", "count", count, "slot", stIt.Slot(), "storage processed", migrdb.GetStorageProcessed(), "current account", migrdb.GetCurrentAccountAddress(), "current account hash", migrdb.GetCurrentAccountHash())
for ; !ts.StorageProcessed && count < maxMovedCount; count++ {
log.Trace("Processing storage", "count", count, "slot", stIt.Slot(), "storage processed", ts.StorageProcessed, "current account", ts.CurrentAccountAddress, "current account hash", ts.GetCurrentAccountHash())
var (
value []byte // slot value after RLP decoding
safeValue [32]byte // 32-byte aligned value
@ -346,12 +345,12 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
}
preimageSeek += int64(len(slotnr))
mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:])
mkv.addStorageSlot(ts.CurrentAccountAddress.Bytes(), slotnr, safeValue[:])
// advance the storage iterator
migrdb.SetStorageProcessed(!stIt.Next())
if !migrdb.GetStorageProcessed() {
migrdb.SetCurrentSlotHash(stIt.Hash())
ts.StorageProcessed = !stIt.Next()
if !ts.StorageProcessed {
ts.CurrentSlotHash = stIt.Hash()
}
}
stIt.Release()
@ -364,19 +363,19 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
if count < maxMovedCount {
count++ // count increase for the account itself
mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc)
mkv.addAccount(ts.CurrentAccountAddress.Bytes(), acc)
// Store the account code if present
if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
code := rawdb.ReadCode(statedb.Database().TrieDB().Disk(), common.BytesToHash(acc.CodeHash))
chunks := trie.ChunkifyCode(code)
mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks)
mkv.addAccountCode(ts.CurrentAccountAddress.Bytes(), uint64(len(code)), chunks)
}
// reset storage iterator marker for next account
migrdb.SetStorageProcessed(false)
migrdb.SetCurrentSlotHash(common.Hash{})
ts.StorageProcessed = false
ts.CurrentSlotHash = common.Hash{}
// Move to the next account, if available - or end
// the transition otherwise.
@ -398,18 +397,18 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
}
log.Trace("Converting account address", "hash", accIt.Hash(), "addr", addr)
preimageSeek += int64(len(addr))
migrdb.SetCurrentAccountAddress(addr)
ts.CurrentAccountAddress = &addr
} else {
// case when the account iterator has
// reached the end but count < maxCount
migrdb.EndVerkleTransition()
ts.Ended = true
break
}
}
}
migrdb.SetCurrentPreimageOffset(preimageSeek)
ts.CurrentPreimageOffset = preimageSeek
log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account hash", statedb.Database().GetCurrentAccountHash(), "last account address", statedb.Database().GetCurrentAccountAddress(), "storage processed", statedb.Database().GetStorageProcessed(), "last storage", statedb.Database().GetCurrentSlotHash())
log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account hash", ts.GetCurrentAccountHash(), "last account address", ts.CurrentAccountAddress, "storage processed", ts.StorageProcessed, "last storage", ts.CurrentSlotHash)
// Take all the collected key-values and prepare the new leaf values.
// This fires a background routine that will start doing the work that
@ -427,5 +426,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
log.Info("Inserted key values in overlay tree", "count", count, "duration", time.Since(now))
}
// TODO(gballet) il faut passer le transition state, sinon je peux pas le sauver après que le root ait été calculé
return nil
}

View file

@ -20,7 +20,7 @@ import (
"bytes"
"encoding/gob"
"fmt"
"sync"
"reflect"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
@ -30,7 +30,6 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/ethereum/go-ethereum/trie/utils"
@ -69,51 +68,9 @@ type Database interface {
// Snapshot returns the underlying state snapshot.
Snapshot() *snapshot.Tree
// StartVerkleTransition marks the start of the verkle transition
StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, verkleTime *uint64, root common.Hash)
SaveTransitionState(common.Hash, *TransitionState)
// EndVerkleTransition marks the end of the verkle transition
EndVerkleTransition()
// InTransition returns true if the verkle transition is currently ongoing
InTransition() bool
// Transitioned returns true if the verkle transition has ended
Transitioned() bool
InitTransitionStatus(bool, bool, common.Hash)
// SetCurrentSlotHash provides the next slot to be translated
SetCurrentSlotHash(common.Hash)
// GetCurrentAccountAddress returns the address of the account that is currently being translated
GetCurrentAccountAddress() *common.Address
SetCurrentAccountAddress(common.Address)
GetCurrentAccountHash() common.Hash
GetCurrentSlotHash() common.Hash
SetStorageProcessed(bool)
GetStorageProcessed() bool
GetCurrentPreimageOffset() int64
SetCurrentPreimageOffset(int64)
AddRootTranslation(originalRoot, translatedRoot common.Hash)
SetLastMerkleRoot(common.Hash)
SaveTransitionState(common.Hash)
LoadTransitionState(common.Hash)
LockCurrentTransitionState()
UnLockCurrentTransitionState()
LoadTransitionState(common.Hash) *TransitionState
}
// Trie is a Ethereum Merkle Patricia trie.
@ -199,24 +156,24 @@ type Trie interface {
type CachingDB struct {
disk ethdb.KeyValueStore
triedb *triedb.Database
verkletriedb *triedb.Database
snap *snapshot.Tree
codeCache *lru.SizeConstrainedCache[common.Hash, []byte]
codeSizeCache *lru.Cache[common.Hash, int]
pointCache *utils.PointCache
// Transition-specific fields
CurrentTransitionState *TransitionState
TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState]
transitionStateLock sync.Mutex
addrToPoint *utils.PointCache
baseRoot common.Hash // hash of last read-only MPT base tree
}
// NewDatabase creates a state database with the provided data sources.
func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB {
func NewDatabase(triedb, verkletriedb *triedb.Database, snap *snapshot.Tree) *CachingDB {
return &CachingDB{
disk: triedb.Disk(),
triedb: triedb,
verkletriedb: verkletriedb,
snap: snap,
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
@ -224,47 +181,6 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB {
}
}
func (db *CachingDB) InTransition() bool {
return db.CurrentTransitionState != nil && db.CurrentTransitionState.Started && !db.CurrentTransitionState.Ended
}
func (db *CachingDB) Transitioned() bool {
return db.CurrentTransitionState != nil && db.CurrentTransitionState.Ended
}
// StartVerkleTransition marks the start of the verkle transition
func (db *CachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, verkleTime *uint64, root common.Hash) {
db.CurrentTransitionState = &TransitionState{
Started: true,
// initialize so that the first storage-less accounts are processed
StorageProcessed: true,
}
// db.AddTranslation(originalRoot, translatedRoot)
db.baseRoot = originalRoot
// Reinitialize values in case of a reorg
if verkleTime != nil {
chainConfig.VerkleTime = verkleTime
}
}
func (db *CachingDB) InitTransitionStatus(started, ended bool, baseRoot common.Hash) {
db.CurrentTransitionState = &TransitionState{
Ended: ended,
Started: started,
// TODO add other fields when we handle mid-transition interrupts
}
db.baseRoot = baseRoot
}
func (db *CachingDB) EndVerkleTransition() {
if !db.CurrentTransitionState.Started {
db.CurrentTransitionState.Started = true
}
db.CurrentTransitionState.Ended = true
}
type TransitionState struct {
CurrentAccountAddress *common.Address // addresss of the last translated account
CurrentSlotHash common.Hash // hash of the last translated storage slot
@ -275,6 +191,16 @@ type TransitionState struct {
// maximum number of leaves of the conversion is reached before the whole storage is
// processed.
StorageProcessed bool
BaseRoot common.Hash // hash of the last read-only MPT base tree
}
func (ts *TransitionState) InTransition() bool {
return ts.Started && !ts.Ended
}
func (ts *TransitionState) Transitioned() bool {
return ts.Ended
}
func (ts *TransitionState) Copy() *TransitionState {
@ -297,7 +223,8 @@ func (ts *TransitionState) Copy() *TransitionState {
// NewDatabaseForTesting is similar to NewDatabase, but it initializes the caching
// db by using an ephemeral memory db with default config for testing.
func NewDatabaseForTesting() *CachingDB {
return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil)
mdb := rawdb.NewMemoryDatabase()
return NewDatabase(triedb.NewDatabase(mdb, nil), triedb.NewDatabase(mdb, triedb.VerkleDefaults), nil)
}
// Reader returns a state reader associated with the specified state root.
@ -322,9 +249,10 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
readers = append(readers, newFlatReader(reader)) // state reader is optional
}
}
ts := db.LoadTransitionState(stateRoot)
// Set up the trie reader, which is expected to always be available
// as the gatekeeper unless the state is corrupted.
tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache, db.InTransition(), db.Transitioned())
tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache, ts)
if err != nil {
return nil, err
}
@ -347,7 +275,8 @@ func (db *CachingDB) openMPTTrie(root common.Hash) (Trie, error) {
// OpenTrie opens the main account trie at a specific root hash.
func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) {
if db.InTransition() || db.Transitioned() {
ts := db.LoadTransitionState(root)
if ts.InTransition() || ts.Transitioned() {
// NOTE this is a kaustinen-only change, it will break replay
vkt, err := trie.NewVerkleTrie(root, db.triedb, db.addrToPoint)
if err != nil {
@ -439,80 +368,49 @@ func (db *CachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point {
return db.addrToPoint.Get(addr)
}
func (db *CachingDB) SetCurrentAccountAddress(addr common.Address) {
db.CurrentTransitionState.CurrentAccountAddress = &addr
}
func (db *CachingDB) GetCurrentAccountHash() common.Hash {
func (ts *TransitionState) GetCurrentAccountHash() common.Hash {
var addrHash common.Hash
if db.CurrentTransitionState.CurrentAccountAddress != nil {
addrHash = crypto.Keccak256Hash(db.CurrentTransitionState.CurrentAccountAddress[:])
if ts.CurrentAccountAddress != nil {
addrHash = crypto.Keccak256Hash(ts.CurrentAccountAddress[:])
}
return addrHash
}
func (db *CachingDB) GetCurrentAccountAddress() *common.Address {
return db.CurrentTransitionState.CurrentAccountAddress
}
func (db *CachingDB) GetCurrentPreimageOffset() int64 {
return db.CurrentTransitionState.CurrentPreimageOffset
}
func (db *CachingDB) SetCurrentPreimageOffset(offset int64) {
db.CurrentTransitionState.CurrentPreimageOffset = offset
}
func (db *CachingDB) SetCurrentSlotHash(hash common.Hash) {
db.CurrentTransitionState.CurrentSlotHash = hash
}
func (db *CachingDB) GetCurrentSlotHash() common.Hash {
return db.CurrentTransitionState.CurrentSlotHash
}
func (db *CachingDB) SetStorageProcessed(processed bool) {
db.CurrentTransitionState.StorageProcessed = processed
}
func (db *CachingDB) GetStorageProcessed() bool {
return db.CurrentTransitionState.StorageProcessed
}
func (db *CachingDB) AddRootTranslation(originalRoot, translatedRoot common.Hash) {
}
func (db *CachingDB) SetLastMerkleRoot(merkleRoot common.Hash) {
db.baseRoot = merkleRoot
}
func (db *CachingDB) SaveTransitionState(root common.Hash) {
db.transitionStateLock.Lock()
defer db.transitionStateLock.Unlock()
if db.CurrentTransitionState != nil {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err := enc.Encode(db.CurrentTransitionState)
if err != nil {
log.Error("failed to encode transition state", "err", err)
return
}
if !db.TransitionStatePerRoot.Contains(root) {
// Copy so that the address pointer isn't updated after
// it has been saved.
db.TransitionStatePerRoot.Add(root, db.CurrentTransitionState.Copy())
rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes())
}
log.Debug("saving transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.Ended, "started", db.CurrentTransitionState.Started)
// SaveTransitionState saves the transition state to the cache and commits
// it to the database if it's not already in the cache.
func (db *CachingDB) SaveTransitionState(root common.Hash, ts *TransitionState) {
if ts == nil {
panic("nil transition state")
}
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
err := enc.Encode(ts)
if err != nil {
log.Error("failed to encode transition state", "err", err)
return
}
if !db.TransitionStatePerRoot.Contains(root) {
// Copy so that the address pointer isn't updated after
// it has been saved.
db.TransitionStatePerRoot.Add(root, ts.Copy())
rawdb.WriteVerkleTransitionState(db.TrieDB().Disk(), root, buf.Bytes())
} else {
// Check that the state is consistent with what is in the cache,
// which is not strictly necessary but a good sanity check. Can
// be removed when the transition is stable.
cachedState, _ := db.TransitionStatePerRoot.Get(root)
if !reflect.DeepEqual(cachedState, ts) {
fmt.Println("transition state mismatch", "cached state", cachedState, "new state", ts)
panic("transition state mismatch")
}
}
log.Debug("saving transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started)
}
func (db *CachingDB) LoadTransitionState(root common.Hash) {
db.transitionStateLock.Lock()
defer db.transitionStateLock.Unlock()
func (db *CachingDB) LoadTransitionState(root common.Hash) *TransitionState {
// Try to get the transition state from the cache and
// the DB if it's not there.
ts, ok := db.TransitionStatePerRoot.Get(root)
@ -521,7 +419,7 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) {
data, err := rawdb.ReadVerkleTransitionState(db.TrieDB().Disk(), root)
if err != nil {
log.Error("failed to read transition state", "err", err)
return
return nil
}
// if a state could be read from the db, attempt to decode it
@ -535,7 +433,7 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) {
err = dec.Decode(&newts)
if err != nil {
log.Error("failed to decode transition state", "err", err)
return
return nil
}
ts = &newts
}
@ -549,19 +447,14 @@ func (db *CachingDB) LoadTransitionState(root common.Hash) {
// Start with a fresh state
ts = &TransitionState{Ended: db.triedb.IsVerkle()}
}
db.TransitionStatePerRoot.Add(root, ts)
}
// Copy so that the CurrentAddress pointer in the map
// doesn't get overwritten.
db.CurrentTransitionState = ts.Copy()
// db.CurrentTransitionState = ts.Copy()
log.Debug("loaded transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.Ended, "started", db.CurrentTransitionState.Started)
}
func (db *CachingDB) LockCurrentTransitionState() {
db.transitionStateLock.Lock()
}
func (db *CachingDB) UnLockCurrentTransitionState() {
db.transitionStateLock.Unlock()
log.Debug("loaded transition state", "storage processed", ts.StorageProcessed, "addr", ts.CurrentAccountAddress, "slot hash", ts.CurrentSlotHash, "root", root, "ended", ts.Ended, "started", ts.Started)
return ts
}

View file

@ -207,12 +207,12 @@ type trieReader struct {
// trieReader constructs a trie reader of the specific state. An error will be
// returned if the associated trie specified by root is not existent.
func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, intransition, transitioned bool) (*trieReader, error) {
func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache, ts *TransitionState) (*trieReader, error) {
var (
tr Trie
err error
)
if !intransition && !transitioned {
if !ts.InTransition() && !ts.Transitioned() {
tr, err = trie.NewStateTrie(trie.StateTrieID(root), db)
} else {
vktr, err := trie.NewVerkleTrie(root, db, cache)
@ -220,7 +220,7 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach
return nil, err
}
tr = vktr
if intransition {
if ts.InTransition() {
mptr, err := trie.NewStateTrie(trie.StateTrieID(root), db)
if err != nil {
return nil, err

View file

@ -40,8 +40,9 @@ func newStateEnv() *stateEnv {
func TestDump(t *testing.T) {
db := rawdb.NewMemoryDatabase()
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
triedb := triedb.NewDatabase(db, &triedb.Config{Preimages: true})
tdb := NewDatabase(triedb, nil)
tdb := NewDatabase(triedb, verkledb, nil)
sdb, _ := New(types.EmptyRootHash, tdb)
s := &stateEnv{state: sdb}

View file

@ -181,7 +181,7 @@ func New(root common.Hash, db Database) (*StateDB, error) {
accessList: newAccessList(),
transientStorage: newTransientStorage(),
}
if db.TrieDB().IsVerkle() {
if tr.IsVerkle() {
sdb.accessEvents = NewAccessEvents(db.PointCache())
}
return sdb, nil

View file

@ -51,7 +51,7 @@ func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *typ
}
// Create and populate the state database to serve as the stateless backend
memdb := witness.MakeHashDB()
db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), nil))
db, err := state.New(witness.Root(), state.NewDatabase(triedb.NewDatabase(memdb, triedb.HashDefaults), triedb.NewDatabase(memdb, triedb.VerkleDefaults), nil))
if err != nil {
return common.Hash{}, common.Hash{}, err
}

View file

@ -69,7 +69,8 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
// please re-enable it for better performance.
tdb := triedb.NewDatabase(eth.chainDb, triedb.HashDefaults)
database = state.NewDatabase(tdb, nil)
vdb := triedb.NewDatabase(eth.chainDb, triedb.VerkleDefaults)
database = state.NewDatabase(tdb, vdb, nil)
if statedb, err = state.New(block.Root(), database); err == nil {
log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number())
return statedb, noopReleaser, nil

View file

@ -36,7 +36,8 @@ func (p *precompileContract) RequiredGas(input []byte) uint64 { return 0 }
func (p *precompileContract) Run(input []byte) ([]byte, error) { return nil, nil }
func TestStateOverrideMovePrecompile(t *testing.T) {
db := state.NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil)
memdb := rawdb.NewMemoryDatabase()
db := state.NewDatabase(triedb.NewDatabase(memdb, nil), triedb.NewDatabase(memdb, nil), nil)
statedb, err := state.New(types.EmptyRootHash, db)
if err != nil {
t.Fatalf("failed to create statedb: %v", err)

View file

@ -135,8 +135,9 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t
if gspec.Config.TerminalTotalDifficulty == nil {
gspec.Config.TerminalTotalDifficulty = big.NewInt(stdmath.MaxInt64)
}
verkledb := triedb.NewDatabase(db, triedb.VerkleDefaults)
triedb := triedb.NewDatabase(db, tconf)
gblock, err := gspec.Commit(db, triedb)
gblock, err := gspec.Commit(db, triedb, verkledb)
if err != nil {
return err
}