core, trie: implement stateless transition

core/overlay/conversion.go

@@ -0,0 +1,434 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package overlay
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/utils"
+	"github.com/ethereum/go-verkle"
+	"github.com/holiman/uint256"
+)
+
+var zeroTreeIndex uint256.Int
+
+// keyValueMigrator is a helper module that collects key-values from the overlay-tree migration for Verkle Trees.
+// It assumes that the walk of the base tree is done in address-order, so it exploit that fact to
+// collect the key-values in a way that is efficient.
+type keyValueMigrator struct {
+	// leafData contains the values for the future leaf for a particular VKT branch.
+	leafData map[branchKey]*migratedKeyValue
+
+	// When prepare() is called, it will start a background routine that will process the leafData
+	// saving the result in newLeaves to be used by migrateCollectedKeyValues(). The background
+	// routine signals that it is done by closing processingReady.
+	processingReady chan struct{}
+	newLeaves       []verkle.LeafNode
+	prepareErr      error
+}
+
+func newKeyValueMigrator() *keyValueMigrator {
+	// We do initialize the VKT config since prepare() might indirectly make multiple GetConfig() calls
+	// in different goroutines when we never called GetConfig() before, causing a race considering the way
+	// that `config` is designed in go-verkle.
+	// TODO: jsign as a fix for this in the PR where we move to a file-less precomp, since it allows safe
+	//       concurrent calls to GetConfig(). When that gets merged, we can remove this line.
+	_ = verkle.GetConfig()
+	return &keyValueMigrator{
+		processingReady: make(chan struct{}),
+		leafData:        make(map[branchKey]*migratedKeyValue, 10_000),
+	}
+}
+
+type migratedKeyValue struct {
+	branchKey    branchKey
+	leafNodeData verkle.BatchNewLeafNodeData
+}
+type branchKey struct {
+	addr      common.Address
+	treeIndex uint256.Int
+}
+
+func newBranchKey(addr []byte, treeIndex *uint256.Int) branchKey {
+	var sk branchKey
+	copy(sk.addr[:], addr)
+	sk.treeIndex = *treeIndex
+	return sk
+}
+
+func (kvm *keyValueMigrator) addStorageSlot(addr []byte, slotNumber []byte, slotValue []byte) {
+	treeIndex, subIndex := utils.StorageIndex(slotNumber)
+	leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, treeIndex))
+	leafNodeData.Values[subIndex] = slotValue
+}
+
+func (kvm *keyValueMigrator) addAccount(addr []byte, acc *types.StateAccount) {
+	leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, &zeroTreeIndex))
+
+	var basicData [verkle.LeafValueSize]byte
+	basicData[utils.BasicDataVersionOffset] = 0
+	binary.BigEndian.PutUint64(basicData[utils.BasicDataNonceOffset:], acc.Nonce)
+
+	// get the lower 16 bytes of water and change its endianness
+	balanceBytes := acc.Balance.Bytes()
+	copy(basicData[32-len(balanceBytes):], balanceBytes[:])
+
+	leafNodeData.Values[utils.BasicDataLeafKey] = basicData[:]
+	leafNodeData.Values[utils.CodeHashLeafKey] = acc.CodeHash[:]
+}
+
+// addAccountCode needs to be called AFTER addAccount, as it will reuse the leaf
+func (kvm *keyValueMigrator) addAccountCode(addr []byte, codeSize uint64, chunks []byte) {
+	leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, &zeroTreeIndex))
+
+	// Save the code size.
+	binary.BigEndian.PutUint32(leafNodeData.Values[utils.BasicDataLeafKey][utils.BasicDataCodeSizeOffset-1:utils.BasicDataNonceOffset], uint32(codeSize))
+
+	// The first 128 chunks are stored in the account header leaf.
+	for i := 0; i < 128 && i < len(chunks)/32; i++ {
+		leafNodeData.Values[byte(128+i)] = chunks[32*i : 32*(i+1)]
+	}
+
+	// Potential further chunks, have their own leaf nodes.
+	for i := 128; i < len(chunks)/32; {
+		treeIndex, _ := utils.CodeChunkIndex(uint256.NewInt(uint64(i)))
+		leafNodeData := kvm.getOrInitLeafNodeData(newBranchKey(addr, treeIndex))
+
+		j := i
+		for ; (j-i) < 256 && j < len(chunks)/32; j++ {
+			leafNodeData.Values[byte((j-128)%256)] = chunks[32*j : 32*(j+1)]
+		}
+		i = j
+	}
+}
+
+func (kvm *keyValueMigrator) getOrInitLeafNodeData(bk branchKey) *verkle.BatchNewLeafNodeData {
+	if ld, ok := kvm.leafData[bk]; ok {
+		return &ld.leafNodeData
+	}
+	kvm.leafData[bk] = &migratedKeyValue{
+		branchKey: bk,
+		leafNodeData: verkle.BatchNewLeafNodeData{
+			Stem:   nil, // It will be calculated in the prepare() phase, since it's CPU heavy.
+			Values: make(map[byte][]byte, 256),
+		},
+	}
+	return &kvm.leafData[bk].leafNodeData
+}
+
+func (kvm *keyValueMigrator) prepare() {
+	// We fire a background routine to process the leafData and save the result in newLeaves.
+	// The background routine signals that it is done by closing processingReady.
+	go func() {
+		// Step 1: We split kvm.leafData in numBatches batches, and we process each batch in a separate goroutine.
+		//         This fills each leafNodeData.Stem with the correct value.
+		leafData := make([]migratedKeyValue, 0, len(kvm.leafData))
+		for _, v := range kvm.leafData {
+			leafData = append(leafData, *v)
+		}
+		var wg sync.WaitGroup
+		batchNum := runtime.NumCPU()
+		batchSize := (len(kvm.leafData) + batchNum - 1) / batchNum
+		for i := 0; i < len(kvm.leafData); i += batchSize {
+			start := i
+			end := i + batchSize
+			if end > len(kvm.leafData) {
+				end = len(kvm.leafData)
+			}
+			wg.Add(1)
+
+			batch := leafData[start:end]
+			go func() {
+				defer wg.Done()
+				var currAddr common.Address
+				var currPoint *verkle.Point
+				for i := range batch {
+					if batch[i].branchKey.addr != currAddr || currAddr == (common.Address{}) {
+						currAddr = batch[i].branchKey.addr
+						currPoint = utils.EvaluateAddressPoint(currAddr[:])
+					}
+					stem := utils.GetTreeKeyWithEvaluatedAddress(currPoint, &batch[i].branchKey.treeIndex, 0)
+					stem = stem[:verkle.StemSize]
+					batch[i].leafNodeData.Stem = stem
+				}
+			}()
+		}
+		wg.Wait()
+
+		// Step 2: Now that we have all stems (i.e: tree keys) calculated, we can create the new leaves.
+		nodeValues := make([]verkle.BatchNewLeafNodeData, len(kvm.leafData))
+		for i := range leafData {
+			nodeValues[i] = leafData[i].leafNodeData
+		}
+
+		// Create all leaves in batch mode so we can optimize cryptography operations.
+		kvm.newLeaves, kvm.prepareErr = verkle.BatchNewLeafNode(nodeValues)
+		close(kvm.processingReady)
+	}()
+}
+
+func (kvm *keyValueMigrator) migrateCollectedKeyValues(tree *trie.VerkleTrie) error {
+	now := time.Now()
+	<-kvm.processingReady
+	if kvm.prepareErr != nil {
+		return fmt.Errorf("failed to prepare key values: %w", kvm.prepareErr)
+	}
+	log.Info("Prepared key values from base tree", "duration", time.Since(now))
+
+	// Insert into the tree.
+	if err := tree.InsertMigratedLeaves(kvm.newLeaves); err != nil {
+		return fmt.Errorf("failed to insert migrated leaves: %w", err)
+	}
+
+	return nil
+}
+
+// OverlayVerkleTransition contains the overlay conversion logic
+func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedCount uint64) error {
+	migrdb := statedb.Database()
+	migrdb.LockCurrentTransitionState()
+	defer migrdb.UnLockCurrentTransitionState()
+
+	// verkle transition: if the conversion process is in progress, move
+	// N values from the MPT into the verkle tree.
+	if migrdb.InTransition() {
+		log.Debug("Processing verkle conversion starting", "account hash", migrdb.GetCurrentAccountHash(), "slot hash", migrdb.GetCurrentSlotHash(), "state root", root)
+		var (
+			now             = time.Now()
+			tt              = statedb.GetTrie().(*trie.TransitionTrie)
+			mpt             = tt.Base()
+			vkt             = tt.Overlay()
+			hasPreimagesBin = false
+			preimageSeek    = migrdb.GetCurrentPreimageOffset()
+			fpreimages      *bufio.Reader
+		)
+
+		// TODO: avoid opening the preimages file here and make it part of, potentially, statedb.Database().
+		filePreimages, err := os.Open("preimages.bin")
+		if err != nil {
+			// fallback on reading the db
+			log.Warn("opening preimage file", "error", err)
+		} else {
+			defer filePreimages.Close()
+			if _, err := filePreimages.Seek(preimageSeek, io.SeekStart); err != nil {
+				return fmt.Errorf("seeking preimage file: %s", err)
+			}
+			fpreimages = bufio.NewReader(filePreimages)
+			hasPreimagesBin = true
+		}
+
+		accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash())
+		if err != nil {
+			return err
+		}
+		defer accIt.Release()
+		accIt.Next()
+
+		// If we're about to start with the migration process, we have to read the first account hash preimage.
+		if migrdb.GetCurrentAccountAddress() == nil {
+			var addr common.Address
+			if hasPreimagesBin {
+				if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
+					return fmt.Errorf("reading preimage file: %s", err)
+				}
+			} else {
+				addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash()))
+				if len(addr) != 20 {
+					return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
+				}
+			}
+			migrdb.SetCurrentAccountAddress(addr)
+			if migrdb.GetCurrentAccountHash() != accIt.Hash() {
+				return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
+			}
+			preimageSeek += int64(len(addr))
+		}
+
+		// mkv will be assiting in the collection of up to maxMovedCount key values to be migrated to the VKT.
+		// It has internal caches to do efficient MPT->VKT key calculations, which will be discarded after
+		// this function.
+		mkv := newKeyValueMigrator()
+		// move maxCount accounts into the verkle tree, starting with the
+		// slots from the previous account.
+		count := uint64(0)
+
+		// if less than maxCount slots were moved, move to the next account
+		for count < maxMovedCount {
+			acc, err := types.FullAccount(accIt.Account())
+			if err != nil {
+				log.Error("Invalid account encountered during traversal", "error", err)
+				return err
+			}
+			vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(), acc.Root)
+
+			// Start with processing the storage, because once the account is
+			// converted, the `stateRoot` field loses its meaning. Which means
+			// that it opens the door to a situation in which the storage isn't
+			// converted, but it can not be found since the account was and so
+			// there is no way to find the MPT storage from the information found
+			// in the verkle account.
+			// Note that this issue can still occur if the account gets written
+			// to during normal block execution. A mitigation strategy has been
+			// introduced with the `*StorageRootConversion` fields in VerkleDB.
+			if len(acc.Root) == 32 && acc.Root != types.EmptyRootHash {
+				stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash())
+				if err != nil {
+					return err
+				}
+				processed := stIt.Next()
+				if processed {
+					log.Debug("account has storage and a next item")
+				} else {
+					log.Debug("account has storage and NO next item")
+				}
+
+				// fdb.StorageProcessed will be initialized to `true` if the
+				// entire storage for an account was not entirely processed
+				// by the previous block. This is used as a signal to resume
+				// processing the storage for that account where we left off.
+				// If the entire storage was processed, then the iterator was
+				// created in vain, but it's ok as this will not happen often.
+				for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ {
+					log.Trace("Processing storage", "count", count, "slot", stIt.Slot(), "storage processed", migrdb.GetStorageProcessed(), "current account", migrdb.GetCurrentAccountAddress(), "current account hash", migrdb.GetCurrentAccountHash())
+					var (
+						value     []byte   // slot value after RLP decoding
+						safeValue [32]byte // 32-byte aligned value
+					)
+					if err := rlp.DecodeBytes(stIt.Slot(), &value); err != nil {
+						return fmt.Errorf("error decoding bytes %x: %w", stIt.Slot(), err)
+					}
+					copy(safeValue[32-len(value):], value)
+
+					var slotnr []byte
+					if hasPreimagesBin {
+						var s [32]byte
+						slotnr = s[:]
+						if _, err := io.ReadFull(fpreimages, slotnr); err != nil {
+							return fmt.Errorf("reading preimage file: %s", err)
+						}
+					} else {
+						slotnr = rawdb.ReadPreimage(migrdb.DiskDB(), stIt.Hash())
+						if len(slotnr) != 32 {
+							return fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr))
+						}
+					}
+					log.Trace("found slot number", "number", slotnr)
+					if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() {
+						return fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr), stIt.Hash())
+					}
+					preimageSeek += int64(len(slotnr))
+
+					mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:])
+
+					// advance the storage iterator
+					migrdb.SetStorageProcessed(!stIt.Next())
+					if !migrdb.GetStorageProcessed() {
+						migrdb.SetCurrentSlotHash(stIt.Hash())
+					}
+				}
+				stIt.Release()
+			}
+
+			// If the maximum number of leaves hasn't been reached, then
+			// it means that the storage has finished processing (or none
+			// was available for this account) and that the account itself
+			// can be processed.
+			if count < maxMovedCount {
+				count++ // count increase for the account itself
+
+				mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc)
+				vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress())
+
+				// Store the account code if present
+				if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
+					code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash))
+					chunks := trie.ChunkifyCode(code)
+
+					mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks)
+				}
+
+				// reset storage iterator marker for next account
+				migrdb.SetStorageProcessed(false)
+				migrdb.SetCurrentSlotHash(common.Hash{})
+
+				// Move to the next account, if available - or end
+				// the transition otherwise.
+				if accIt.Next() {
+					log.Trace("Found another account to convert", "hash", accIt.Hash())
+					var addr common.Address
+					if hasPreimagesBin {
+						if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
+							return fmt.Errorf("reading preimage file: %s", err)
+						}
+					} else {
+						addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash()))
+						if len(addr) != 20 {
+							return fmt.Errorf("account address len is zero is not 20: %d", len(addr))
+						}
+					}
+					if crypto.Keccak256Hash(addr[:]) != accIt.Hash() {
+						return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
+					}
+					log.Trace("Converting account address", "hash", accIt.Hash(), "addr", addr)
+					preimageSeek += int64(len(addr))
+					migrdb.SetCurrentAccountAddress(addr)
+				} else {
+					// case when the account iterator has
+					// reached the end but count < maxCount
+					migrdb.EndVerkleTransition()
+					break
+				}
+			}
+		}
+		migrdb.SetCurrentPreimageOffset(preimageSeek)
+
+		log.Info("Collected key values from base tree", "count", count, "duration", time.Since(now), "last account hash", statedb.Database().GetCurrentAccountHash(), "last account address", statedb.Database().GetCurrentAccountAddress(), "storage processed", statedb.Database().GetStorageProcessed(), "last storage", statedb.Database().GetCurrentSlotHash())
+
+		// Take all the collected key-values and prepare the new leaf values.
+		// This fires a background routine that will start doing the work that
+		// migrateCollectedKeyValues() will use to insert into the tree.
+		//
+		// TODO: Now both prepare() and migrateCollectedKeyValues() are next to each other, but
+		//       after we fix an existing bug, we can call prepare() before the block execution and
+		//       let it do the work in the background. After the block execution and finalization
+		//       finish, we can call migrateCollectedKeyValues() which should already find everything ready.
+		mkv.prepare()
+		now = time.Now()
+		if err := mkv.migrateCollectedKeyValues(tt.Overlay()); err != nil {
+			return fmt.Errorf("could not migrate key values: %w", err)
+		}
+		log.Info("Inserted key values in overlay tree", "count", count, "duration", time.Since(now))
+	}
+
+	return nil
+}

core/state/database.go

@@ -17,7 +17,10 @@
 package state
 
 import (
+	"bytes"
+	"encoding/gob"
 	"fmt"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
@@ -26,10 +29,13 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/trie/utils"
 	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/ethereum/go-verkle"
 )
 
 const (
@@ -62,6 +68,46 @@ type Database interface {
 
 	// Snapshot returns the underlying state snapshot.
 	Snapshot() *snapshot.Tree
+
+	StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, verkleTime *uint64, root common.Hash)
+
+	EndVerkleTransition()
+
+	InTransition() bool
+
+	Transitioned() bool
+
+	InitTransitionStatus(bool, bool, common.Hash)
+
+	SetCurrentSlotHash(common.Hash)
+
+	GetCurrentAccountAddress() *common.Address
+
+	SetCurrentAccountAddress(common.Address)
+
+	GetCurrentAccountHash() common.Hash
+
+	GetCurrentSlotHash() common.Hash
+
+	SetStorageProcessed(bool)
+
+	GetStorageProcessed() bool
+
+	GetCurrentPreimageOffset() int64
+
+	SetCurrentPreimageOffset(int64)
+
+	AddRootTranslation(originalRoot, translatedRoot common.Hash)
+
+	SetLastMerkleRoot(common.Hash)
+
+	SaveTransitionState(common.Hash)
+
+	LoadTransitionState(common.Hash)
+
+	LockCurrentTransitionState()
+
+	UnLockCurrentTransitionState()
 }
 
 // Trie is a Ethereum Merkle Patricia trie.
@@ -151,6 +197,13 @@ type CachingDB struct {
 	codeCache     *lru.SizeConstrainedCache[common.Hash, []byte]
 	codeSizeCache *lru.Cache[common.Hash, int]
 	pointCache    *utils.PointCache
+
+	// Transition-specific fields
+	CurrentTransitionState *TransitionState
+	TransitionStatePerRoot lru.BasicLRU[common.Hash, *TransitionState]
+	transitionStateLock    sync.Mutex
+	addrToPoint            *utils.PointCache
+	baseRoot               common.Hash // hash of last read-only MPT base tree
 }
 
 // NewDatabase creates a state database with the provided data sources.
@@ -165,6 +218,75 @@ func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB {
 	}
 }
 
+func (db *CachingDB) InTransition() bool {
+	return db.CurrentTransitionState != nil && db.CurrentTransitionState.Started && !db.CurrentTransitionState.Ended
+}
+
+func (db *CachingDB) Transitioned() bool {
+	return db.CurrentTransitionState != nil && db.CurrentTransitionState.Ended
+}
+
+func (db *CachingDB) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, verkleTime *uint64, root common.Hash) {
+	db.CurrentTransitionState = &TransitionState{
+		Started: true,
+		// initialize so that the first storage-less accounts are processed
+		StorageProcessed: true,
+	}
+	// db.AddTranslation(originalRoot, translatedRoot)
+	db.baseRoot = originalRoot
+
+	// Reinitialize values in case of a reorg
+	if verkleTime != nil {
+		chainConfig.VerkleTime = verkleTime
+	}
+}
+
+func (db *CachingDB) InitTransitionStatus(started, ended bool, baseRoot common.Hash) {
+	db.CurrentTransitionState = &TransitionState{
+		Ended:   ended,
+		Started: started,
+		// TODO add other fields when we handle mid-transition interrupts
+	}
+	db.baseRoot = baseRoot
+}
+
+func (db *CachingDB) EndVerkleTransition() {
+	if !db.CurrentTransitionState.Started {
+		db.CurrentTransitionState.Started = true
+	}
+
+	db.CurrentTransitionState.Ended = true
+}
+
+type TransitionState struct {
+	CurrentAccountAddress *common.Address // addresss of the last translated account
+	CurrentSlotHash       common.Hash     // hash of the last translated storage slot
+	CurrentPreimageOffset int64           // next byte to read from the preimage file
+	Started, Ended        bool
+
+	// Mark whether the storage for an account has been processed. This is useful if the
+	// maximum number of leaves of the conversion is reached before the whole storage is
+	// processed.
+	StorageProcessed bool
+}
+
+func (ts *TransitionState) Copy() *TransitionState {
+	ret := &TransitionState{
+		Started:               ts.Started,
+		Ended:                 ts.Ended,
+		CurrentSlotHash:       ts.CurrentSlotHash,
+		CurrentPreimageOffset: ts.CurrentPreimageOffset,
+		StorageProcessed:      ts.StorageProcessed,
+	}
+
+	if ts.CurrentAccountAddress != nil {
+		ret.CurrentAccountAddress = &common.Address{}
+		copy(ret.CurrentAccountAddress[:], ts.CurrentAccountAddress[:])
+	}
+
+	return ret
+}
+
 // NewDatabaseForTesting is similar to NewDatabase, but it initializes the caching
 // db by using an ephemeral memory db with default config for testing.
 func NewDatabaseForTesting() *CachingDB {
@@ -208,11 +330,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) {
 	return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil
 }
 
-// OpenTrie opens the main account trie at a specific root hash.
+func (db *CachingDB) openMPTTrie(root common.Hash) (Trie, error) {
-func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) {
-	if db.triedb.IsVerkle() {
-		return trie.NewVerkleTrie(root, db.triedb, db.pointCache)
-	}
 	tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb)
 	if err != nil {
 		return nil, err
@@ -220,6 +338,51 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) {
 	return tr, nil
 }
 
+func (db *CachingDB) openVKTrie(_ common.Hash) (Trie, error) {
+	payload, err := db.DiskDB().Get(trie.FlatDBVerkleNodeKeyPrefix)
+	if err != nil {
+		return trie.NewVerkleTrie(verkle.New(), db.triedb, db.addrToPoint, db.CurrentTransitionState.Ended)
+	}
+
+	r, err := verkle.ParseNode(payload, 0)
+	if err != nil {
+		panic(err)
+	}
+	return trie.NewVerkleTrie(r, db.triedb, db.addrToPoint, db.CurrentTransitionState.Ended)
+}
+
+// OpenTrie opens the main account trie at a specific root hash.
+func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) {
+	if db.InTransition() || db.Transitioned() {
+		// NOTE this is a kaustinen-only change, it will break replay
+		vkt, err := db.openVKTrie(root)
+		if err != nil {
+			log.Error("failed to open the vkt", "err", err)
+			return nil, err
+		}
+
+		// If the verkle conversion has ended, return a single
+		// verkle trie.
+		if db.CurrentTransitionState.Ended {
+			log.Debug("transition ended, returning a simple verkle tree")
+			return vkt, nil
+		}
+
+		// Otherwise, return a transition trie, with a base MPT
+		// trie and an overlay, verkle trie.
+		mpt, err := db.openMPTTrie(db.baseRoot)
+		if err != nil {
+			log.Error("failed to open the mpt", "err", err, "root", db.baseRoot)
+			return nil, err
+		}
+
+		return trie.NewTransitionTree(mpt.(*trie.SecureTrie), vkt.(*trie.VerkleTrie), false), nil
+	}
+
+	log.Info("not in transition, opening mpt alone", "root", root)
+	return db.openMPTTrie(root)
+}
+
 // OpenStorageTrie opens the storage trie of an account.
 func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) {
 	// In the verkle case, there is only one tree. But the two-tree structure
@@ -277,3 +440,134 @@ func mustCopyTrie(t Trie) Trie {
 		panic(fmt.Errorf("unknown trie type %T", t))
 	}
 }
+
+func (db *CachingDB) GetTreeKeyHeader(addr []byte) *verkle.Point {
+	return db.addrToPoint.Get(addr)
+}
+
+func (db *CachingDB) SetCurrentAccountAddress(addr common.Address) {
+	db.CurrentTransitionState.CurrentAccountAddress = &addr
+}
+
+func (db *CachingDB) GetCurrentAccountHash() common.Hash {
+	var addrHash common.Hash
+	if db.CurrentTransitionState.CurrentAccountAddress != nil {
+		addrHash = crypto.Keccak256Hash(db.CurrentTransitionState.CurrentAccountAddress[:])
+	}
+	return addrHash
+}
+
+func (db *CachingDB) GetCurrentAccountAddress() *common.Address {
+	return db.CurrentTransitionState.CurrentAccountAddress
+}
+
+func (db *CachingDB) GetCurrentPreimageOffset() int64 {
+	return db.CurrentTransitionState.CurrentPreimageOffset
+}
+
+func (db *CachingDB) SetCurrentPreimageOffset(offset int64) {
+	db.CurrentTransitionState.CurrentPreimageOffset = offset
+}
+
+func (db *CachingDB) SetCurrentSlotHash(hash common.Hash) {
+	db.CurrentTransitionState.CurrentSlotHash = hash
+}
+
+func (db *CachingDB) GetCurrentSlotHash() common.Hash {
+	return db.CurrentTransitionState.CurrentSlotHash
+}
+
+func (db *CachingDB) SetStorageProcessed(processed bool) {
+	db.CurrentTransitionState.StorageProcessed = processed
+}
+
+func (db *CachingDB) GetStorageProcessed() bool {
+	return db.CurrentTransitionState.StorageProcessed
+}
+
+func (db *CachingDB) AddRootTranslation(originalRoot, translatedRoot common.Hash) {
+}
+
+func (db *CachingDB) SetLastMerkleRoot(merkleRoot common.Hash) {
+	db.baseRoot = merkleRoot
+}
+
+func (db *CachingDB) SaveTransitionState(root common.Hash) {
+	db.transitionStateLock.Lock()
+	defer db.transitionStateLock.Unlock()
+	if db.CurrentTransitionState != nil {
+		var buf bytes.Buffer
+		enc := gob.NewEncoder(&buf)
+		err := enc.Encode(db.CurrentTransitionState)
+		if err != nil {
+			log.Error("failed to encode transition state", "err", err)
+			return
+		}
+
+		if !db.TransitionStatePerRoot.Contains(root) {
+			// Copy so that the address pointer isn't updated after
+			// it has been saved.
+			db.TransitionStatePerRoot.Add(root, db.CurrentTransitionState.Copy())
+
+			rawdb.WriteVerkleTransitionState(db.DiskDB(), root, buf.Bytes())
+		}
+
+		log.Debug("saving transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.Ended, "started", db.CurrentTransitionState.Started)
+	}
+}
+
+func (db *CachingDB) LoadTransitionState(root common.Hash) {
+	db.transitionStateLock.Lock()
+	defer db.transitionStateLock.Unlock()
+	// Try to get the transition state from the cache and
+	// the DB if it's not there.
+	ts, ok := db.TransitionStatePerRoot.Get(root)
+	if !ok {
+		// Not in the cache, try getting it from the DB
+		data, err := rawdb.ReadVerkleTransitionState(db.DiskDB(), root)
+		if err != nil {
+			log.Error("failed to read transition state", "err", err)
+			return
+		}
+
+		// if a state could be read from the db, attempt to decode it
+		if len(data) > 0 {
+			var (
+				newts TransitionState
+				buf   = bytes.NewBuffer(data[:])
+				dec   = gob.NewDecoder(buf)
+			)
+			// Decode transition state
+			err = dec.Decode(&newts)
+			if err != nil {
+				log.Error("failed to decode transition state", "err", err)
+				return
+			}
+			ts = &newts
+		}
+
+		// Fallback that should only happen before the transition
+		if ts == nil {
+			// Initialize the first transition state, with the "ended"
+			// field set to true if the database was created
+			// as a verkle database.
+			log.Debug("no transition state found, starting fresh", "is verkle", db.triedb.IsVerkle())
+			// Start with a fresh state
+			ts = &TransitionState{Ended: db.triedb.IsVerkle()}
+		}
+	}
+
+	// Copy so that the CurrentAddress pointer in the map
+	// doesn't get overwritten.
+	db.CurrentTransitionState = ts.Copy()
+
+	log.Debug("loaded transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.Ended, "started", db.CurrentTransitionState.Started)
+}
+
+func (db *CachingDB) LockCurrentTransitionState() {
+	db.transitionStateLock.Lock()
+}
+
+func (db *CachingDB) UnLockCurrentTransitionState() {
+	db.transitionStateLock.Unlock()
+}

core/state_processor.go

@@ -22,11 +22,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/misc"
+	"github.com/ethereum/go-ethereum/core/overlay"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -121,6 +123,14 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	// Finalize the block, applying any consensus engine specific extras (e.g. block rewards)
 	p.chain.engine.Finalize(p.chain, header, tracingStateDB, block.Body())
 
+	if p.chain.Config().IsVerkle(header.Number, header.Time) {
+		parent := p.chain.GetHeaderByHash(header.ParentHash)
+		if err := overlay.OverlayVerkleTransition(statedb, parent.Root, p.chain.Config().OverlayStride); err != nil {
+			log.Error("error performing the transition", "err", err)
+			panic(fmt.Sprintf("error performing the transition: %s", err))
+		}
+	}
+
 	return &ProcessResult{
 		Receipts: receipts,
 		Requests: requests,

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/cespare/cp v0.1.0
 	github.com/cloudflare/cloudflare-go v0.114.0
 	github.com/cockroachdb/pebble v1.1.2
-	github.com/consensys/gnark-crypto v0.14.0
+	github.com/consensys/gnark-crypto v0.17.0
 	github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a
 	github.com/crate-crypto/go-kzg-4844 v1.1.0
 	github.com/davecgh/go-spew v1.1.1
@@ -22,7 +22,7 @@ require (
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0
 	github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3
 	github.com/ethereum/c-kzg-4844 v1.0.0
-	github.com/ethereum/go-verkle v0.2.2
+	github.com/ethereum/go-verkle v0.2.1
 	github.com/fatih/color v1.16.0
 	github.com/ferranbt/fastssz v0.1.2
 	github.com/fjl/gencodec v0.1.0
@@ -66,8 +66,8 @@ require (
 	go.uber.org/goleak v1.3.0
 	golang.org/x/crypto v0.35.0
 	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
-	golang.org/x/sync v0.11.0
+	golang.org/x/sync v0.12.0
-	golang.org/x/sys v0.30.0
+	golang.org/x/sys v0.31.0
 	golang.org/x/text v0.22.0
 	golang.org/x/time v0.9.0
 	golang.org/x/tools v0.29.0
@@ -91,14 +91,14 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect
 	github.com/aws/smithy-go v1.15.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.17.0 // indirect
+	github.com/bits-and-blooms/bitset v1.22.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cockroachdb/errors v1.11.3 // indirect
 	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/consensys/bavard v0.1.22 // indirect
+	github.com/consensys/bavard v0.1.30 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
 	github.com/deepmap/oapi-codegen v1.6.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect

go.sum

@@ -90,8 +90,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI=
+github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4=
-github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
@@ -124,10 +124,10 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
-github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A=
+github.com/consensys/bavard v0.1.30 h1:wwAj9lSnMLFXjEclKwyhf7Oslg8EoaFz9u1QGgt0bsk=
-github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs=
+github.com/consensys/bavard v0.1.30/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs=
-github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E=
+github.com/consensys/gnark-crypto v0.17.0 h1:vKDhZMOrySbpZDCvGMOELrHFv/A9mJ7+9I8HEfRZSkI=
-github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0=
+github.com/consensys/gnark-crypto v0.17.0/go.mod h1:A2URlMHUT81ifJ0UlLzSlm7TmnE3t7VxEThApdMukJw=
 github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
 github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg=
@@ -166,8 +166,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA=
 github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
-github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8=
+github.com/ethereum/go-verkle v0.2.1 h1:FYWEByFT19jT/ym/dy++E+f1SSw899uXGNrhCkwjYJw=
-github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
+github.com/ethereum/go-verkle v0.2.1/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk=
@@ -640,8 +640,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -705,8 +705,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

params/config.go

@@ -432,6 +432,9 @@ type ChainConfig struct {
 	Ethash             *EthashConfig       `json:"ethash,omitempty"`
 	Clique             *CliqueConfig       `json:"clique,omitempty"`
 	BlobScheduleConfig *BlobScheduleConfig `json:"blobSchedule,omitempty"`
+
+	// Stateless transition conversion rules
+	OverlayStride uint64 `json:"overlayStride,omitempty"`
 }
 
 // EthashConfig is the consensus engine configs for proof-of-work based sealing.

trie/transition.go

@@ -0,0 +1,194 @@
+// Copyright 2021 go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-verkle"
+)
+
+type TransitionTrie struct {
+	overlay *VerkleTrie
+	base    *SecureTrie
+	storage bool
+}
+
+func NewTransitionTree(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie {
+	return &TransitionTrie{
+		overlay: overlay,
+		base:    base,
+		storage: st,
+	}
+}
+
+func (t *TransitionTrie) Base() *SecureTrie {
+	return t.base
+}
+
+// TODO(gballet/jsign): consider removing this API.
+func (t *TransitionTrie) Overlay() *VerkleTrie {
+	return t.overlay
+}
+
+// GetKey returns the sha3 preimage of a hashed key that was previously used
+// to store a value.
+//
+// TODO(fjl): remove this when StateTrie is removed
+func (t *TransitionTrie) GetKey(key []byte) []byte {
+	if key := t.overlay.GetKey(key); key != nil {
+		return key
+	}
+	return t.base.GetKey(key)
+}
+
+// Get returns the value for key stored in the trie. The value bytes must
+// not be modified by the caller. If a node was not found in the database, a
+// trie.MissingNodeError is returned.
+func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) {
+	val, err := t.overlay.GetStorage(addr, key)
+	if err != nil {
+		return nil, fmt.Errorf("get storage from overlay: %s", err)
+	}
+	if len(val) != 0 {
+		return val, nil
+	}
+	// TODO also insert value into overlay
+	return t.base.GetStorage(addr, key)
+}
+
+// GetAccount abstract an account read from the trie.
+func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount, error) {
+	data, err := t.overlay.GetAccount(address)
+	if err != nil {
+		return nil, err
+	}
+	if data != nil {
+		if t.overlay.db.HasStorageRootConversion(address) {
+			data.Root = t.overlay.db.StorageRootConversion(address)
+		}
+		return data, nil
+	}
+	// TODO also insert value into overlay
+	return t.base.GetAccount(address)
+}
+
+// Update associates key with value in the trie. If value has length zero, any
+// existing value is deleted from the trie. The value bytes must not be modified
+// by the caller while they are stored in the trie. If a node was not found in the
+// database, a trie.MissingNodeError is returned.
+func (t *TransitionTrie) UpdateStorage(address common.Address, key []byte, value []byte) error {
+	var v []byte
+	if len(value) >= 32 {
+		v = value[:32]
+	} else {
+		var val [32]byte
+		copy(val[32-len(value):], value[:])
+		v = val[:]
+	}
+	return t.overlay.UpdateStorage(address, key, v)
+}
+
+// UpdateAccount abstract an account write to the trie.
+func (t *TransitionTrie) UpdateAccount(addr common.Address, account *types.StateAccount, codeLen int) error {
+	if account.Root != (common.Hash{}) && account.Root != types.EmptyRootHash {
+		t.overlay.db.SetStorageRootConversion(addr, account.Root)
+	}
+	return t.overlay.UpdateAccount(addr, account, codeLen)
+}
+
+// Delete removes any existing value for key from the trie. If a node was not
+// found in the database, a trie.MissingNodeError is returned.
+func (t *TransitionTrie) DeleteStorage(addr common.Address, key []byte) error {
+	return t.overlay.DeleteStorage(addr, key)
+}
+
+// DeleteAccount abstracts an account deletion from the trie.
+func (t *TransitionTrie) DeleteAccount(key common.Address) error {
+	return t.overlay.DeleteAccount(key)
+}
+
+// Hash returns the root hash of the trie. It does not write to the database and
+// can be used even if the trie doesn't have one.
+func (t *TransitionTrie) Hash() common.Hash {
+	return t.overlay.Hash()
+}
+
+// Commit collects all dirty nodes in the trie and replace them with the
+// corresponding node hash. All collected nodes(including dirty leaves if
+// collectLeaf is true) will be encapsulated into a nodeset for return.
+// The returned nodeset can be nil if the trie is clean(nothing to commit).
+// Once the trie is committed, it's not usable anymore. A new trie must
+// be created with new root and updated trie database for following usage
+func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
+	// Just return if the trie is a storage trie: otherwise,
+	// the overlay trie will be committed as many times as
+	// there are storage tries. This would kill performance.
+	if t.storage {
+		return common.Hash{}, nil
+	}
+	return t.overlay.Commit(collectLeaf)
+}
+
+// NodeIterator returns an iterator that returns nodes of the trie. Iteration
+// starts at the key after the given start key.
+func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) {
+	panic("not implemented") // TODO: Implement
+}
+
+// Prove constructs a Merkle proof for key. The result contains all encoded nodes
+// on the path to the value at key. The value itself is also included in the last
+// node and can be retrieved by verifying the proof.
+//
+// If the trie does not contain a value for key, the returned proof contains all
+// nodes of the longest existing prefix of the key (at least the root), ending
+// with the node that proves the absence of the key.
+func (t *TransitionTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
+	panic("not implemented") // TODO: Implement
+}
+
+// IsVerkle returns true if the trie is verkle-tree based
+func (t *TransitionTrie) IsVerkle() bool {
+	// For all intents and purposes, the calling code should treat this as a verkle trie
+	return true
+}
+
+func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error {
+	trie := t.overlay
+	switch root := trie.root.(type) {
+	case *verkle.InternalNode:
+		return root.InsertValuesAtStem(key, values, t.overlay.FlatdbNodeResolver)
+	default:
+		panic("invalid root type")
+	}
+}
+
+func (t *TransitionTrie) Copy() *TransitionTrie {
+	return &TransitionTrie{
+		overlay: t.overlay.Copy(),
+		base:    t.base.Copy(),
+		storage: t.storage,
+	}
+}
+
+func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error {
+	return t.overlay.UpdateContractCode(addr, codeHash, code)
+}

trie/utils/verkle.go

@@ -185,7 +185,7 @@ func CodeHashKey(address []byte) []byte {
 	return GetTreeKey(address, zero, CodeHashLeafKey)
 }
 
-func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) {
+func CodeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) {
 	var (
 		chunkOffset            = new(uint256.Int).Add(codeOffset, chunk)
 		treeIndex, subIndexMod = new(uint256.Int).DivMod(chunkOffset, verkleNodeWidth, new(uint256.Int))
@@ -196,7 +196,7 @@ func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) {
 // CodeChunkKey returns the verkle tree key of the code chunk for the
 // specified account.
 func CodeChunkKey(address []byte, chunk *uint256.Int) []byte {
-	treeIndex, subIndex := codeChunkIndex(chunk)
+	treeIndex, subIndex := CodeChunkIndex(chunk)
 	return GetTreeKey(address, treeIndex, subIndex)
 }
 
@@ -260,7 +260,7 @@ func CodeHashKeyWithEvaluatedAddress(evaluated *verkle.Point) []byte {
 // chunk for the specified account. The difference between CodeChunkKey is the
 // address evaluation is already computed to minimize the computational overhead.
 func CodeChunkKeyWithEvaluatedAddress(addressPoint *verkle.Point, chunk *uint256.Int) []byte {
-	treeIndex, subIndex := codeChunkIndex(chunk)
+	treeIndex, subIndex := CodeChunkIndex(chunk)
 	return GetTreeKeyWithEvaluatedAddress(addressPoint, treeIndex, subIndex)
 }
 

trie/verkle.go

@@ -428,3 +428,7 @@ func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) {
 func (t *VerkleTrie) Witness() map[string]struct{} {
 	panic("not implemented")
 }
+
+func (trie *VerkleTrie) InsertMigratedLeaves(leaves []verkle.LeafNode) error {
+	return trie.root.(*verkle.InternalNode).InsertMigratedLeaves(leaves, trie.FlatdbNodeResolver)
+}