forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

verkle without overlay transition

Browse source
This commit is contained in:
Guillaume Ballet 2023-08-15 12:32:22 +02:00
parent 40b939021e
commit 888e2f5f38
15 changed files with 9 additions and 874 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/CODEOWNERS vendored
View file

@ -6,7 +6,7 @@ accounts/scwallet @gballet
accounts/abi @gballet @MariusVanDerWijden
cmd/clef @holiman
consensus @karalabe
core/ @karalabe @rjl493456442
core/ @karalabe @holiman @rjl493456442
eth/ @karalabe @holiman @rjl493456442
eth/catalyst/ @gballet
eth/tracers/ @s1na

48
.github/workflows/go.yml vendored
View file

@ -1,48 +0,0 @@
name: Go lint and test
on:
push:
branches: [ master ]
pull_request:
branches: [ master, verkle-trie-proof-in-block-rebased, verkle-trie-post-merge, beverly-hills-head, 'verkle/replay-change-with-tree-group-tryupdate' ]
workflow_dispatch:
jobs:
build:
runs-on: self-hosted
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
- name: Build
run: go build -v ./...
lint:
runs-on: self-hosted
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
- name: Download golangci-lint
run: wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s latest
- name: Lint
run: ./bin/golangci-lint run
- name: Vet
run: go vet
test:
runs-on: self-hosted
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.18
- name: Download precomputed points
run: wget -nv https://github.com/gballet/go-verkle/releases/download/banderwagonv3/precomp -Otrie/utils/precomp
- name: Test
run: go test ./...

38
cmd/geth/chaincmd.go
View file

@ -144,17 +144,6 @@ It's deprecated, please use "geth db import" instead.
Description: `
The export-preimages command exports hash preimages to an RLP encoded stream.
It's deprecated, please use "geth db export" instead.
`,
}
exportOverlayPreimagesCommand = &cli.Command{
Action: exportOverlayPreimages,
Name: "export-overlay-preimages",
Usage: "Export the preimage in overlay tree migration order",
ArgsUsage: "<dumpfile>",
Flags: flags.Merge([]cli.Flag{utils.TreeRootFlag}, utils.DatabasePathFlags),
Description: `
The export-overlay-preimages command exports hash preimages to a flat file, in exactly
the expected order for the overlay tree migration.
`,
}
dumpCommand = &cli.Command{
@ -410,33 +399,6 @@ func exportPreimages(ctx *cli.Context) error {
return nil
}
// exportOverlayPreimages dumps the preimage data to a flat file.
func exportOverlayPreimages(ctx *cli.Context) error {
if ctx.Args().Len() < 1 {
utils.Fatalf("This command requires an argument.")
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, _ := utils.MakeChain(ctx, stack, true)
var root common.Hash
if ctx.String(utils.TreeRootFlag.Name) != "" {
rootBytes := common.FromHex(ctx.String(utils.StartKeyFlag.Name))
if len(rootBytes) != common.HashLength {
return fmt.Errorf("invalid root hash length")
}
root = common.BytesToHash(rootBytes)
}
start := time.Now()
if err := utils.ExportOverlayPreimages(chain, ctx.Args().First(), root); err != nil {
utils.Fatalf("Export error: %v\n", err)
}
fmt.Printf("Export done in %v\n", time.Since(start))
return nil
}
func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
db := utils.MakeChainDatabase(ctx, stack, true)
var header *types.Header

6
cmd/geth/main.go
View file

@ -39,14 +39,12 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/node"
"go.uber.org/automaxprocs/maxprocs"
// Force-load the tracer engines to trigger registration
_ "github.com/ethereum/go-ethereum/eth/tracers/js"
_ "github.com/ethereum/go-ethereum/eth/tracers/native"
// Automatically set GOMAXPROCS to match Linux container CPU quota.
_ "go.uber.org/automaxprocs"
"github.com/urfave/cli/v2"
)
@ -209,7 +207,6 @@ func init() {
exportCommand,
importPreimagesCommand,
exportPreimagesCommand,
exportOverlayPreimagesCommand,
removedbCommand,
dumpCommand,
dumpGenesisCommand,
@ -246,6 +243,7 @@ func init() {
)
app.Before = func(ctx *cli.Context) error {
maxprocs.Set() // Automatically set GOMAXPROCS to match Linux container CPU quota.
flags.MigrateGlobalFlags(ctx)
return debug.Setup(ctx)
}

81
cmd/utils/cmd.go
View file

@ -176,18 +176,6 @@ func ImportChain(chain *core.BlockChain, fn string) error {
return err
}
}
// cpuProfile, err := os.Create("cpu.out")
// if err != nil {
// return fmt.Errorf("Error creating CPU profile: %v", err)
// }
// defer cpuProfile.Close()
// err = pprof.StartCPUProfile(cpuProfile)
// if err != nil {
// return fmt.Errorf("Error starting CPU profile: %v", err)
// }
// defer pprof.StopCPUProfile()
// params.ClearVerkleWitnessCosts()
stream := rlp.NewStream(reader, 0)
// Run actual the import.
@ -386,75 +374,6 @@ func ExportPreimages(db ethdb.Database, fn string) error {
return nil
}
// ExportOverlayPreimages exports all known hash preimages into the specified file,
// in the same order as expected by the overlay tree migration.
func ExportOverlayPreimages(chain *core.BlockChain, fn string, root common.Hash) error {
log.Info("Exporting preimages", "file", fn)
fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
if err != nil {
return err
}
defer fh.Close()
writer := bufio.NewWriter(fh)
defer writer.Flush()
statedb, err := chain.State()
if err != nil {
return fmt.Errorf("failed to open statedb: %w", err)
}
if root == (common.Hash{}) {
root = chain.CurrentBlock().Root
}
accIt, err := statedb.Snaps().AccountIterator(root, common.Hash{})
if err != nil {
return err
}
defer accIt.Release()
count := 0
for accIt.Next() {
acc, err := types.FullAccount(accIt.Account())
if err != nil {
return fmt.Errorf("invalid account encountered during traversal: %s", err)
}
addr := rawdb.ReadPreimage(statedb.Database().DiskDB(), accIt.Hash())
if len(addr) != 20 {
return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
}
if _, err := writer.Write(addr); err != nil {
return fmt.Errorf("failed to write addr preimage: %w", err)
}
if acc.HasStorage() {
stIt, err := statedb.Snaps().StorageIterator(root, accIt.Hash(), common.Hash{})
if err != nil {
return fmt.Errorf("failed to create storage iterator: %w", err)
}
for stIt.Next() {
slotnr := rawdb.ReadPreimage(statedb.Database().DiskDB(), stIt.Hash())
if len(slotnr) != 32 {
return fmt.Errorf("slotnr not 32 len")
}
if _, err := writer.Write(slotnr); err != nil {
return fmt.Errorf("failed to write slotnr preimage: %w", err)
}
}
stIt.Release()
}
count++
if count%100000 == 0 {
log.Info("Last exported account", "account", accIt.Hash())
}
}
log.Info("Exported preimages", "file", fn)
return nil
}
// exportHeader is used in the export/import flow. When we do an export,
// the first element we output is the exportHeader.
// Whenever a backwards-incompatible change is made, the Version header

5
cmd/utils/flags.go
View file

@ -216,11 +216,6 @@ var (
Usage: "Max number of elements (0 = no limit)",
Value: 0,
}
TreeRootFlag = &cli.StringFlag{
Name: "roothash",
Usage: "Root hash of the tree (if empty, use the latest)",
Value: "",
}
defaultSyncMode = ethconfig.Defaults.SyncMode
SyncModeFlag = &flags.TextMarshalerFlag{

1
core/block_validator.go
View file

@ -102,7 +102,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
return consensus.ErrUnknownAncestor
}
fmt.Println("failure here")
return consensus.ErrPrunedAncestor
}
return nil

45
core/blockchain.go
View file

@ -18,15 +18,11 @@
package core
import (
"bufio"
"errors"
"fmt"
"io"
"math"
"math/big"
"os"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
@ -1519,30 +1515,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
return bc.insertChain(chain, true)
}
func findVerkleConversionBlock() (uint64, error) {
if _, err := os.Stat("conversion.txt"); os.IsNotExist(err) {
return math.MaxUint64, nil
}
f, err := os.Open("conversion.txt")
if err != nil {
log.Error("Failed to open conversion.txt", "err", err)
return 0, err
}
defer f.Close()
scanner := bufio.NewScanner(f)
scanner.Scan()
conversionBlock, err := strconv.ParseUint(scanner.Text(), 10, 64)
if err != nil {
log.Error("Failed to parse conversionBlock", "err", err)
return 0, err
}
log.Info("Found conversion block info", "conversionBlock", conversionBlock)
return conversionBlock, nil
}
// insertChain is the internal implementation of InsertChain, which assumes that
// 1) chains are contiguous, and 2) The chain mutex is held.
//
@ -1557,11 +1529,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
return 0, nil
}
conversionBlock, err := findVerkleConversionBlock()
if err != nil {
return 0, err
}
// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
SenderCacher.RecoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number(), chain[0].Time()), chain)
@ -1744,10 +1711,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
}
if parent.Number.Uint64() == conversionBlock {
bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time)
bc.stateCache.SetLastMerkleRoot(parent.Root)
}
statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps)
if err != nil {
return it.index, err
@ -2531,14 +2494,6 @@ func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
return time.Duration(bc.flushInterval.Load())
}
func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64) {
bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, cancunTime)
}
func (bc *BlockChain) EndVerkleTransition() {
bc.stateCache.EndVerkleTransition()
}
func (bc *BlockChain) AddRootTranslation(originalRoot, translatedRoot common.Hash) {
bc.stateCache.AddRootTranslation(originalRoot, translatedRoot)
}

80
core/state/database.go
View file

@ -69,8 +69,6 @@ type Database interface {
EndVerkleTransition()
InTransition() bool
Transitioned() bool
SetCurrentSlotHash(hash common.Hash)
@ -92,8 +90,6 @@ type Database interface {
SetCurrentPreimageOffset(int64)
AddRootTranslation(originalRoot, translatedRoot common.Hash)
SetLastMerkleRoot(root common.Hash)
}
// Trie is a Ethereum Merkle Patricia trie.
@ -201,10 +197,6 @@ func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database {
}
}
func (db *cachingDB) InTransition() bool {
return db.started && !db.ended
}
func (db *cachingDB) Transitioned() bool {
return db.ended
}
@ -312,47 +304,16 @@ func (db *cachingDB) openVKTrie(root common.Hash) (Trie, error) {
// OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
var (
mpt Trie
err error
)
// TODO separate both cases when I can be certain that it won't
// find a Verkle trie where is expects a Transitoion trie.
if db.started || db.ended {
var r common.Hash
if db.ended {
r = root
} else {
r = db.getTranslation(root)
}
vkt, err := db.openVKTrie(r)
if db.ended {
vkt, err := db.openVKTrie(root)
if err != nil {
return nil, err
}
// If the verkle conversion has ended, return a single
// verkle trie.
if db.ended {
return vkt, nil
}
// Otherwise, return a transition trie, with a base MPT
// trie and an overlay, verkle trie.
mpt, err = db.openMPTTrie(db.baseRoot)
if err != nil {
return nil, err
}
return trie.NewTransitionTree(mpt.(*trie.SecureTrie), vkt.(*trie.VerkleTrie), false), nil
} else {
mpt, err = db.openMPTTrie(root)
if err != nil {
return nil, err
}
return vkt, nil
}
return mpt, nil
return db.openMPTTrie(root)
}
func (db *cachingDB) openStorageMPTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) {
@ -366,36 +327,7 @@ func (db *cachingDB) openStorageMPTrie(stateRoot common.Hash, address common.Add
// OpenStorageTrie opens the storage trie of an account
func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) {
if db.ended {
mpt, err := db.openStorageMPTrie(common.Hash{}, address, common.Hash{}, self)
if err != nil {
return nil, err
}
// Return a "storage trie" that is an adapter between the storge MPT
// and the unique verkle tree.
switch self := self.(type) {
case *trie.VerkleTrie:
return trie.NewTransitionTree(mpt.(*trie.StateTrie), self, true), nil
case *trie.TransitionTrie:
return trie.NewTransitionTree(mpt.(*trie.StateTrie), self.Overlay(), true), nil
default:
panic("unexpected trie type")
}
}
if db.started {
mpt, err := db.openStorageMPTrie(db.LastMerkleRoot, address, root, nil)
if err != nil {
return nil, err
}
// Return a "storage trie" that is an adapter between the storge MPT
// and the unique verkle tree.
switch self := self.(type) {
case *trie.VerkleTrie:
return trie.NewTransitionTree(mpt.(*trie.SecureTrie), self, true), nil
case *trie.TransitionTrie:
return trie.NewTransitionTree(mpt.(*trie.SecureTrie), self.Overlay(), true), nil
default:
panic("unexpected trie type")
}
return self, nil
}
mpt, err := db.openStorageMPTrie(stateRoot, address, root, nil)
return mpt, err
@ -406,8 +338,6 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
switch t := t.(type) {
case *trie.StateTrie:
return t.Copy()
case *trie.TransitionTrie:
return t.Copy()
case *trie.VerkleTrie:
return t.Copy()
default:

9
core/state/statedb.go
View file

@ -1051,14 +1051,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now())
}
root := s.trie.Hash()
// Save the root of the MPT so that it can be used during the transition
if !s.Database().InTransition() && !s.Database().Transitioned() {
s.Database().SetLastMerkleRoot(root)
}
return root
return s.trie.Hash()
}
// SetTxContext sets the current transaction hash and index which are

329
core/state_processor.go
View file

@ -17,31 +17,18 @@
package core
import (
"bufio"
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"math/big"
"os"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/misc"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
tutils "github.com/ethereum/go-ethereum/trie/utils"
"github.com/gballet/go-verkle"
"github.com/holiman/uint256"
)
// StateProcessor is a basic Processor, which takes care of transitioning
@ -109,211 +96,9 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
return nil, nil, 0, errors.New("withdrawals before shanghai")
}
// Overlay tree migration logic
migrdb := statedb.Database()
// verkle transition: if the conversion process is in progress, move
// N values from the MPT into the verkle tree.
if migrdb.InTransition() {
var (
now = time.Now()
tt = statedb.GetTrie().(*trie.TransitionTrie)
mpt = tt.Base()
vkt = tt.Overlay()
hasPreimagesBin = false
preimageSeek = migrdb.GetCurrentPreimageOffset()
fpreimages *bufio.Reader
)
// TODO: avoid opening the preimages file here and make it part of, potentially, statedb.Database().
filePreimages, err := os.Open("preimages.bin")
if err != nil {
// fallback on reading the db
log.Warn("opening preimage file", "error", err)
} else {
defer filePreimages.Close()
if _, err := filePreimages.Seek(preimageSeek, io.SeekStart); err != nil {
return nil, nil, 0, fmt.Errorf("seeking preimage file: %s", err)
}
fpreimages = bufio.NewReader(filePreimages)
hasPreimagesBin = true
}
accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash())
if err != nil {
return nil, nil, 0, err
}
defer accIt.Release()
accIt.Next()
// If we're about to start with the migration process, we have to read the first account hash preimage.
if migrdb.GetCurrentAccountAddress() == nil {
var addr common.Address
if hasPreimagesBin {
if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err)
}
} else {
addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash()))
if len(addr) != 20 {
return nil, nil, 0, fmt.Errorf("addr len is zero is not 32: %d", len(addr))
}
}
migrdb.SetCurrentAccountAddress(addr)
if migrdb.GetCurrentAccountHash() != accIt.Hash() {
return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
}
preimageSeek += int64(len(addr))
}
const maxMovedCount = 10000
// mkv will be assiting in the collection of up to maxMovedCount key values to be migrated to the VKT.
// It has internal caches to do efficient MPT->VKT key calculations, which will be discarded after
// this function.
mkv := &keyValueMigrator{vktLeafData: make(map[string]*verkle.BatchNewLeafNodeData)}
// move maxCount accounts into the verkle tree, starting with the
// slots from the previous account.
count := 0
// if less than maxCount slots were moved, move to the next account
for count < maxMovedCount {
acc, err := types.FullAccount(accIt.Account())
if err != nil {
log.Error("Invalid account encountered during traversal", "error", err)
return nil, nil, 0, err
}
vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(), acc.Root)
// Start with processing the storage, because once the account is
// converted, the `stateRoot` field loses its meaning. Which means
// that it opens the door to a situation in which the storage isn't
// converted, but it can not be found since the account was and so
// there is no way to find the MPT storage from the information found
// in the verkle account.
// Note that this issue can still occur if the account gets written
// to during normal block execution. A mitigation strategy has been
// introduced with the `*StorageRootConversion` fields in VerkleDB.
if acc.HasStorage() {
stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash())
if err != nil {
return nil, nil, 0, err
}
stIt.Next()
// fdb.StorageProcessed will be initialized to `true` if the
// entire storage for an account was not entirely processed
// by the previous block. This is used as a signal to resume
// processing the storage for that account where we left off.
// If the entire storage was processed, then the iterator was
// created in vain, but it's ok as this will not happen often.
for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ {
var (
value []byte // slot value after RLP decoding
safeValue [32]byte // 32-byte aligned value
)
if err := rlp.DecodeBytes(stIt.Slot(), &value); err != nil {
return nil, nil, 0, fmt.Errorf("error decoding bytes %x: %w", stIt.Slot(), err)
}
copy(safeValue[32-len(value):], value)
var slotnr []byte
if hasPreimagesBin {
var s [32]byte
slotnr = s[:]
if _, err := io.ReadFull(fpreimages, slotnr); err != nil {
return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err)
}
} else {
slotnr = rawdb.ReadPreimage(migrdb.DiskDB(), stIt.Hash())
if len(slotnr) != 32 {
return nil, nil, 0, fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr))
}
}
if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() {
return nil, nil, 0, fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr), stIt.Hash())
}
preimageSeek += int64(len(slotnr))
mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:])
// advance the storage iterator
migrdb.SetStorageProcessed(!stIt.Next())
if !migrdb.GetStorageProcessed() {
migrdb.SetCurrentSlotHash(stIt.Hash())
}
}
stIt.Release()
}
// If the maximum number of leaves hasn't been reached, then
// it means that the storage has finished processing (or none
// was available for this account) and that the account itself
// can be processed.
if count < maxMovedCount {
count++ // count increase for the account itself
mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc)
vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress())
// Store the account code if present
if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash))
chunks := trie.ChunkifyCode(code)
mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks)
}
// reset storage iterator marker for next account
migrdb.SetStorageProcessed(false)
migrdb.SetCurrentSlotHash(common.Hash{})
// Move to the next account, if available - or end
// the transition otherwise.
if accIt.Next() {
var addr common.Address
if hasPreimagesBin {
if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err)
}
} else {
addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash()))
if len(addr) != 20 {
return nil, nil, 0, fmt.Errorf("account address len is zero is not 20: %d", len(addr))
}
}
// fmt.Printf("account switch: %s != %s\n", crypto.Keccak256Hash(addr[:]), accIt.Hash())
if crypto.Keccak256Hash(addr[:]) != accIt.Hash() {
return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
}
preimageSeek += int64(len(addr))
migrdb.SetCurrentAccountAddress(addr)
} else {
// case when the account iterator has
// reached the end but count < maxCount
migrdb.EndVerkleTransition()
break
}
}
}
migrdb.SetCurrentPreimageOffset(preimageSeek)
log.Info("Collected and prepared key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash())
now = time.Now()
if err := mkv.migrateCollectedKeyValues(tt.Overlay()); err != nil {
return nil, nil, 0, fmt.Errorf("could not migrate key values: %w", err)
}
log.Info("Inserted key values in overlay tree", "count", count, "duration", time.Since(now))
}
// Finalize the block, applying any consensus engine specific extras (e.g. block rewards)
p.engine.Finalize(p.bc, header, statedb, block.Transactions(), block.Uncles(), withdrawals)
if block.NumberU64()%100 == 0 {
stateRoot := statedb.GetTrie().Hash()
log.Info("State root", "number", block.NumberU64(), "hash", stateRoot)
}
return receipts, allLogs, *usedGas, nil
}
@ -379,117 +164,3 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo
vmenv := vm.NewEVM(blockContext, vm.TxContext{BlobHashes: tx.BlobHashes()}, statedb, config, cfg)
return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv)
}
// keyValueMigrator is a helper struct that collects key-values from the base tree.
// The walk is done in account order, so **we assume** the APIs hold this invariant. This is
// useful to be smart about caching banderwagon.Points to make VKT key calculations faster.
type keyValueMigrator struct {
currAddr []byte
currAddrPoint *verkle.Point
vktLeafData map[string]*verkle.BatchNewLeafNodeData
}
func (kvm *keyValueMigrator) addStorageSlot(addr []byte, slotNumber []byte, slotValue []byte) {
addrPoint := kvm.getAddrPoint(addr)
vktKey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotNumber)
leafNodeData := kvm.getOrInitLeafNodeData(vktKey)
leafNodeData.Values[vktKey[verkle.StemSize]] = slotValue
}
func (kvm *keyValueMigrator) addAccount(addr []byte, acc *types.StateAccount) {
addrPoint := kvm.getAddrPoint(addr)
vktKey := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint)
leafNodeData := kvm.getOrInitLeafNodeData(vktKey)
var version [verkle.LeafValueSize]byte
leafNodeData.Values[tutils.VersionLeafKey] = version[:]
var balance [verkle.LeafValueSize]byte
for i, b := range acc.Balance.Bytes() {
balance[len(acc.Balance.Bytes())-1-i] = b
}
leafNodeData.Values[tutils.BalanceLeafKey] = balance[:]
var nonce [verkle.LeafValueSize]byte
binary.LittleEndian.PutUint64(nonce[:8], acc.Nonce)
leafNodeData.Values[tutils.NonceLeafKey] = nonce[:]
leafNodeData.Values[tutils.CodeKeccakLeafKey] = acc.CodeHash[:]
// Code size is ignored here. If this isn't an EOA, the tree-walk will call
// addAccountCode with this information.
}
func (kvm *keyValueMigrator) addAccountCode(addr []byte, codeSize uint64, chunks []byte) {
addrPoint := kvm.getAddrPoint(addr)
vktKey := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint)
leafNodeData := kvm.getOrInitLeafNodeData(vktKey)
// Save the code size.
var codeSizeBytes [verkle.LeafValueSize]byte
binary.LittleEndian.PutUint64(codeSizeBytes[:8], codeSize)
leafNodeData.Values[tutils.CodeSizeLeafKey] = codeSizeBytes[:]
// The first 128 chunks are stored in the account header leaf.
for i := 0; i < 128 && i < len(chunks)/32; i++ {
leafNodeData.Values[byte(128+i)] = chunks[32*i : 32*(i+1)]
}
// Potential further chunks, have their own leaf nodes.
for i := 128; i < len(chunks)/32; {
vktKey := tutils.GetTreeKeyCodeChunkWithEvaluatedAddress(addrPoint, uint256.NewInt(uint64(i)))
leafNodeData := kvm.getOrInitLeafNodeData(vktKey)
j := i
for ; (j-i) < 256 && j < len(chunks)/32; j++ {
leafNodeData.Values[byte((j-128)%256)] = chunks[32*j : 32*(j+1)]
}
i = j
}
}
func (kvm *keyValueMigrator) getAddrPoint(addr []byte) *verkle.Point {
if bytes.Equal(addr, kvm.currAddr) {
return kvm.currAddrPoint
}
kvm.currAddr = addr
kvm.currAddrPoint = tutils.EvaluateAddressPoint(addr)
return kvm.currAddrPoint
}
func (kvm *keyValueMigrator) getOrInitLeafNodeData(stem []byte) *verkle.BatchNewLeafNodeData {
stemStr := string(stem)
if _, ok := kvm.vktLeafData[stemStr]; !ok {
kvm.vktLeafData[stemStr] = &verkle.BatchNewLeafNodeData{
Stem: stem[:verkle.StemSize],
Values: make(map[byte][]byte),
}
}
return kvm.vktLeafData[stemStr]
}
func (kvm *keyValueMigrator) migrateCollectedKeyValues(tree *trie.VerkleTrie) error {
// Transform the map into a slice.
nodeValues := make([]verkle.BatchNewLeafNodeData, 0, len(kvm.vktLeafData))
for _, vld := range kvm.vktLeafData {
nodeValues = append(nodeValues, *vld)
}
// Create all leaves in batch mode so we can optimize cryptography operations.
newLeaves, err := verkle.BatchNewLeafNode(nodeValues)
if err != nil {
return fmt.Errorf("failed to batch-create new leaf nodes")
}
// Insert into the tree.
if err := tree.InsertMigratedLeaves(newLeaves); err != nil {
return fmt.Errorf("failed to insert migrated leaves: %w", err)
}
return nil
}

4
light/trie.go
View file

@ -109,10 +109,6 @@ func (db *odrDatabase) EndVerkleTransition() {
panic("not implemented") // TODO: Implement
}
func (db *odrDatabase) InTransition() bool {
panic("not implemented") // TODO: Implement
}
func (db *odrDatabase) Transitioned() bool {
panic("not implemented") // TODO: Implement
}

34
trie/database.go
View file

@ -236,40 +236,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
return hdb.Node(hash)
}
func (db *Database) HasStorageRootConversion(addr common.Address) bool {
db.addrToRootLock.RLock()
defer db.addrToRootLock.RUnlock()
if db.addrToRoot == nil {
return false
}
_, ok := db.addrToRoot[addr]
return ok
}
func (db *Database) SetStorageRootConversion(addr common.Address, root common.Hash) {
db.addrToRootLock.Lock()
defer db.addrToRootLock.Unlock()
if db.addrToRoot == nil {
db.addrToRoot = make(map[common.Address]common.Hash)
}
db.addrToRoot[addr] = root
}
func (db *Database) StorageRootConversion(addr common.Address) common.Hash {
db.addrToRootLock.RLock()
defer db.addrToRootLock.RUnlock()
if db.addrToRoot == nil {
return common.Hash{}
}
return db.addrToRoot[addr]
}
func (db *Database) ClearStorageRootConversion(addr common.Address) {
db.addrToRootLock.Lock()
defer db.addrToRootLock.Unlock()
delete(db.addrToRoot, addr)
}
func (db *Database) IsVerkle() bool {
return db.config != nil && db.config.Verkle
}

193
trie/transition.go
View file

@ -1,193 +0,0 @@
// Copyright 2021 go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie/trienode"
"github.com/gballet/go-verkle"
)
type TransitionTrie struct {
overlay *VerkleTrie
base *SecureTrie
storage bool
}
func NewTransitionTree(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie {
return &TransitionTrie{
overlay: overlay,
base: base,
storage: st,
}
}
func (t *TransitionTrie) Base() *SecureTrie {
return t.base
}
// TODO(gballet/jsign): consider removing this API.
func (t *TransitionTrie) Overlay() *VerkleTrie {
return t.overlay
}
// GetKey returns the sha3 preimage of a hashed key that was previously used
// to store a value.
//
// TODO(fjl): remove this when StateTrie is removed
func (t *TransitionTrie) GetKey(key []byte) []byte {
if key := t.overlay.GetKey(key); key != nil {
return key
}
return t.base.GetKey(key)
}
// Get returns the value for key stored in the trie. The value bytes must
// not be modified by the caller. If a node was not found in the database, a
// trie.MissingNodeError is returned.
func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) {
if val, err := t.overlay.GetStorage(addr, key); len(val) != 0 || err != nil {
return val, nil
}
// TODO also insert value into overlay
return t.base.GetStorage(addr, key)
}
// GetAccount abstract an account read from the trie.
func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount, error) {
data, err := t.overlay.GetAccount(address)
if err != nil {
// WORKAROUND, see the definition of errDeletedAccount
// for an explainer of why this if is needed.
if err == errDeletedAccount {
return nil, nil
}
return nil, err
}
if data != nil {
if t.overlay.db.HasStorageRootConversion(address) {
data.Root = t.overlay.db.StorageRootConversion(address)
}
return data, nil
}
// TODO also insert value into overlay
return t.base.GetAccount(address)
}
// Update associates key with value in the trie. If value has length zero, any
// existing value is deleted from the trie. The value bytes must not be modified
// by the caller while they are stored in the trie. If a node was not found in the
// database, a trie.MissingNodeError is returned.
func (t *TransitionTrie) UpdateStorage(address common.Address, key []byte, value []byte) error {
var v []byte
if len(value) >= 32 {
v = value[:32]
} else {
var val [32]byte
copy(val[32-len(value):], value[:])
v = val[:]
}
return t.overlay.UpdateStorage(address, key, v)
}
// UpdateAccount abstract an account write to the trie.
func (t *TransitionTrie) UpdateAccount(addr common.Address, account *types.StateAccount) error {
if account.Root != (common.Hash{}) && account.Root != types.EmptyRootHash {
t.overlay.db.SetStorageRootConversion(addr, account.Root)
}
return t.overlay.UpdateAccount(addr, account)
}
// Delete removes any existing value for key from the trie. If a node was not
// found in the database, a trie.MissingNodeError is returned.
func (t *TransitionTrie) DeleteStorage(addr common.Address, key []byte) error {
return t.overlay.DeleteStorage(addr, key)
}
// DeleteAccount abstracts an account deletion from the trie.
func (t *TransitionTrie) DeleteAccount(key common.Address) error {
return t.overlay.DeleteAccount(key)
}
// Hash returns the root hash of the trie. It does not write to the database and
// can be used even if the trie doesn't have one.
func (t *TransitionTrie) Hash() common.Hash {
return t.overlay.Hash()
}
// Commit collects all dirty nodes in the trie and replace them with the
// corresponding node hash. All collected nodes(including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
// The returned nodeset can be nil if the trie is clean(nothing to commit).
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) {
// Just return if the trie is a storage trie: otherwise,
// the overlay trie will be committed as many times as
// there are storage tries. This would kill performance.
if t.storage {
return common.Hash{}, nil, nil
}
return t.overlay.Commit(collectLeaf)
}
// NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key.
func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) {
panic("not implemented") // TODO: Implement
}
// Prove constructs a Merkle proof for key. The result contains all encoded nodes
// on the path to the value at key. The value itself is also included in the last
// node and can be retrieved by verifying the proof.
//
// If the trie does not contain a value for key, the returned proof contains all
// nodes of the longest existing prefix of the key (at least the root), ending
// with the node that proves the absence of the key.
func (t *TransitionTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
panic("not implemented") // TODO: Implement
}
// IsVerkle returns true if the trie is verkle-tree based
func (t *TransitionTrie) IsVerkle() bool {
// For all intents and purposes, the calling code should treat this as a verkle trie
return true
}
func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error {
trie := t.overlay
switch root := trie.root.(type) {
case *verkle.InternalNode:
return root.InsertStem(key, values, t.overlay.flatdbNodeResolver)
default:
panic("invalid root type")
}
}
func (t *TransitionTrie) Copy() *TransitionTrie {
return &TransitionTrie{
overlay: t.overlay.Copy(),
base: t.base.Copy(),
storage: t.storage,
}
}
func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error {
return t.overlay.UpdateContractCode(addr, codeHash, code)
}

8
trie/verkle.go
View file

@ -469,14 +469,6 @@ func ChunkifyCode(code []byte) ChunkedCode {
return chunks
}
func (t *VerkleTrie) SetStorageRootConversion(addr common.Address, root common.Hash) {
t.db.SetStorageRootConversion(addr, root)
}
func (t *VerkleTrie) ClearStrorageRootConversion(addr common.Address) {
t.db.ClearStorageRootConversion(addr)
}
func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error {
var (
chunks = ChunkifyCode(code)