diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f38d522b7e..faf922df01 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,7 +6,7 @@ accounts/scwallet @gballet accounts/abi @gballet @MariusVanDerWijden cmd/clef @holiman consensus @karalabe -core/ @karalabe @rjl493456442 +core/ @karalabe @holiman @rjl493456442 eth/ @karalabe @holiman @rjl493456442 eth/catalyst/ @gballet eth/tracers/ @s1na diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 5ae526f1ee..0000000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Go lint and test - -on: - push: - branches: [ master ] - pull_request: - branches: [ master, verkle-trie-proof-in-block-rebased, verkle-trie-post-merge, beverly-hills-head, 'verkle/replay-change-with-tree-group-tryupdate' ] - workflow_dispatch: - -jobs: - build: - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.18 - - name: Build - run: go build -v ./... - - lint: - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.18 - - name: Download golangci-lint - run: wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s latest - - name: Lint - run: ./bin/golangci-lint run - - name: Vet - run: go vet - - test: - runs-on: self-hosted - steps: - - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.18 - - name: Download precomputed points - run: wget -nv https://github.com/gballet/go-verkle/releases/download/banderwagonv3/precomp -Otrie/utils/precomp - - name: Test - run: go test ./... diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 8e195bcf96..41591ac138 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -144,17 +144,6 @@ It's deprecated, please use "geth db import" instead. Description: ` The export-preimages command exports hash preimages to an RLP encoded stream. It's deprecated, please use "geth db export" instead. -`, - } - exportOverlayPreimagesCommand = &cli.Command{ - Action: exportOverlayPreimages, - Name: "export-overlay-preimages", - Usage: "Export the preimage in overlay tree migration order", - ArgsUsage: "", - Flags: flags.Merge([]cli.Flag{utils.TreeRootFlag}, utils.DatabasePathFlags), - Description: ` -The export-overlay-preimages command exports hash preimages to a flat file, in exactly -the expected order for the overlay tree migration. `, } dumpCommand = &cli.Command{ @@ -410,33 +399,6 @@ func exportPreimages(ctx *cli.Context) error { return nil } -// exportOverlayPreimages dumps the preimage data to a flat file. -func exportOverlayPreimages(ctx *cli.Context) error { - if ctx.Args().Len() < 1 { - utils.Fatalf("This command requires an argument.") - } - stack, _ := makeConfigNode(ctx) - defer stack.Close() - - chain, _ := utils.MakeChain(ctx, stack, true) - - var root common.Hash - if ctx.String(utils.TreeRootFlag.Name) != "" { - rootBytes := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) - if len(rootBytes) != common.HashLength { - return fmt.Errorf("invalid root hash length") - } - root = common.BytesToHash(rootBytes) - } - - start := time.Now() - if err := utils.ExportOverlayPreimages(chain, ctx.Args().First(), root); err != nil { - utils.Fatalf("Export error: %v\n", err) - } - fmt.Printf("Export done in %v\n", time.Since(start)) - return nil -} - func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { db := utils.MakeChainDatabase(ctx, stack, true) var header *types.Header diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a239f88499..568aec4b8b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -39,14 +39,12 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "go.uber.org/automaxprocs/maxprocs" // Force-load the tracer engines to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" - // Automatically set GOMAXPROCS to match Linux container CPU quota. - _ "go.uber.org/automaxprocs" - "github.com/urfave/cli/v2" ) @@ -209,7 +207,6 @@ func init() { exportCommand, importPreimagesCommand, exportPreimagesCommand, - exportOverlayPreimagesCommand, removedbCommand, dumpCommand, dumpGenesisCommand, @@ -246,6 +243,7 @@ func init() { ) app.Before = func(ctx *cli.Context) error { + maxprocs.Set() // Automatically set GOMAXPROCS to match Linux container CPU quota. flags.MigrateGlobalFlags(ctx) return debug.Setup(ctx) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 24da4911bc..16b1260572 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -176,18 +176,6 @@ func ImportChain(chain *core.BlockChain, fn string) error { return err } } - // cpuProfile, err := os.Create("cpu.out") - // if err != nil { - // return fmt.Errorf("Error creating CPU profile: %v", err) - // } - // defer cpuProfile.Close() - // err = pprof.StartCPUProfile(cpuProfile) - // if err != nil { - // return fmt.Errorf("Error starting CPU profile: %v", err) - // } - // defer pprof.StopCPUProfile() - // params.ClearVerkleWitnessCosts() - stream := rlp.NewStream(reader, 0) // Run actual the import. @@ -386,75 +374,6 @@ func ExportPreimages(db ethdb.Database, fn string) error { return nil } -// ExportOverlayPreimages exports all known hash preimages into the specified file, -// in the same order as expected by the overlay tree migration. -func ExportOverlayPreimages(chain *core.BlockChain, fn string, root common.Hash) error { - log.Info("Exporting preimages", "file", fn) - - fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) - if err != nil { - return err - } - defer fh.Close() - - writer := bufio.NewWriter(fh) - defer writer.Flush() - - statedb, err := chain.State() - if err != nil { - return fmt.Errorf("failed to open statedb: %w", err) - } - - if root == (common.Hash{}) { - root = chain.CurrentBlock().Root - } - - accIt, err := statedb.Snaps().AccountIterator(root, common.Hash{}) - if err != nil { - return err - } - defer accIt.Release() - - count := 0 - for accIt.Next() { - acc, err := types.FullAccount(accIt.Account()) - if err != nil { - return fmt.Errorf("invalid account encountered during traversal: %s", err) - } - addr := rawdb.ReadPreimage(statedb.Database().DiskDB(), accIt.Hash()) - if len(addr) != 20 { - return fmt.Errorf("addr len is zero is not 32: %d", len(addr)) - } - if _, err := writer.Write(addr); err != nil { - return fmt.Errorf("failed to write addr preimage: %w", err) - } - - if acc.HasStorage() { - stIt, err := statedb.Snaps().StorageIterator(root, accIt.Hash(), common.Hash{}) - if err != nil { - return fmt.Errorf("failed to create storage iterator: %w", err) - } - for stIt.Next() { - slotnr := rawdb.ReadPreimage(statedb.Database().DiskDB(), stIt.Hash()) - if len(slotnr) != 32 { - return fmt.Errorf("slotnr not 32 len") - } - if _, err := writer.Write(slotnr); err != nil { - return fmt.Errorf("failed to write slotnr preimage: %w", err) - } - } - stIt.Release() - } - count++ - if count%100000 == 0 { - log.Info("Last exported account", "account", accIt.Hash()) - } - } - - log.Info("Exported preimages", "file", fn) - return nil -} - // exportHeader is used in the export/import flow. When we do an export, // the first element we output is the exportHeader. // Whenever a backwards-incompatible change is made, the Version header diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c92f49d432..e0c7a42670 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -216,11 +216,6 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } - TreeRootFlag = &cli.StringFlag{ - Name: "roothash", - Usage: "Root hash of the tree (if empty, use the latest)", - Value: "", - } defaultSyncMode = ethconfig.Defaults.SyncMode SyncModeFlag = &flags.TextMarshalerFlag{ diff --git a/core/block_validator.go b/core/block_validator.go index b1ceab9d5c..3c9ac3dc49 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -102,7 +102,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) { return consensus.ErrUnknownAncestor } - fmt.Println("failure here") return consensus.ErrPrunedAncestor } return nil diff --git a/core/blockchain.go b/core/blockchain.go index a1b3b6b6cb..0b90eca6ff 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,15 +18,11 @@ package core import ( - "bufio" "errors" "fmt" "io" - "math" "math/big" - "os" "runtime" - "strconv" "strings" "sync" "sync/atomic" @@ -1519,30 +1515,6 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return bc.insertChain(chain, true) } -func findVerkleConversionBlock() (uint64, error) { - if _, err := os.Stat("conversion.txt"); os.IsNotExist(err) { - return math.MaxUint64, nil - } - - f, err := os.Open("conversion.txt") - if err != nil { - log.Error("Failed to open conversion.txt", "err", err) - return 0, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - scanner.Scan() - conversionBlock, err := strconv.ParseUint(scanner.Text(), 10, 64) - if err != nil { - log.Error("Failed to parse conversionBlock", "err", err) - return 0, err - } - log.Info("Found conversion block info", "conversionBlock", conversionBlock) - - return conversionBlock, nil -} - // insertChain is the internal implementation of InsertChain, which assumes that // 1) chains are contiguous, and 2) The chain mutex is held. // @@ -1557,11 +1529,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) return 0, nil } - conversionBlock, err := findVerkleConversionBlock() - if err != nil { - return 0, err - } - // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) SenderCacher.RecoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number(), chain[0].Time()), chain) @@ -1744,10 +1711,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1) } - if parent.Number.Uint64() == conversionBlock { - bc.StartVerkleTransition(parent.Root, emptyVerkleRoot, bc.Config(), &parent.Time) - bc.stateCache.SetLastMerkleRoot(parent.Root) - } statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) if err != nil { return it.index, err @@ -2531,14 +2494,6 @@ func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } -func (bc *BlockChain) StartVerkleTransition(originalRoot, translatedRoot common.Hash, chainConfig *params.ChainConfig, cancunTime *uint64) { - bc.stateCache.StartVerkleTransition(originalRoot, translatedRoot, chainConfig, cancunTime) -} - func (bc *BlockChain) EndVerkleTransition() { bc.stateCache.EndVerkleTransition() } - -func (bc *BlockChain) AddRootTranslation(originalRoot, translatedRoot common.Hash) { - bc.stateCache.AddRootTranslation(originalRoot, translatedRoot) -} diff --git a/core/state/database.go b/core/state/database.go index 7ee8489d3f..2747f5e5d0 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -69,8 +69,6 @@ type Database interface { EndVerkleTransition() - InTransition() bool - Transitioned() bool SetCurrentSlotHash(hash common.Hash) @@ -92,8 +90,6 @@ type Database interface { SetCurrentPreimageOffset(int64) AddRootTranslation(originalRoot, translatedRoot common.Hash) - - SetLastMerkleRoot(root common.Hash) } // Trie is a Ethereum Merkle Patricia trie. @@ -201,10 +197,6 @@ func NewDatabaseWithNodeDB(db ethdb.Database, triedb *trie.Database) Database { } } -func (db *cachingDB) InTransition() bool { - return db.started && !db.ended -} - func (db *cachingDB) Transitioned() bool { return db.ended } @@ -312,47 +304,16 @@ func (db *cachingDB) openVKTrie(root common.Hash) (Trie, error) { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - var ( - mpt Trie - err error - ) - // TODO separate both cases when I can be certain that it won't - // find a Verkle trie where is expects a Transitoion trie. - if db.started || db.ended { - var r common.Hash - if db.ended { - r = root - } else { - r = db.getTranslation(root) - } - vkt, err := db.openVKTrie(r) + if db.ended { + vkt, err := db.openVKTrie(root) if err != nil { return nil, err } - // If the verkle conversion has ended, return a single - // verkle trie. - if db.ended { - return vkt, nil - } - - // Otherwise, return a transition trie, with a base MPT - // trie and an overlay, verkle trie. - mpt, err = db.openMPTTrie(db.baseRoot) - if err != nil { - return nil, err - } - - return trie.NewTransitionTree(mpt.(*trie.SecureTrie), vkt.(*trie.VerkleTrie), false), nil - } else { - mpt, err = db.openMPTTrie(root) - if err != nil { - return nil, err - } + return vkt, nil } - - return mpt, nil + return db.openMPTTrie(root) } func (db *cachingDB) openStorageMPTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) { @@ -366,36 +327,7 @@ func (db *cachingDB) openStorageMPTrie(stateRoot common.Hash, address common.Add // OpenStorageTrie opens the storage trie of an account func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { if db.ended { - mpt, err := db.openStorageMPTrie(common.Hash{}, address, common.Hash{}, self) - if err != nil { - return nil, err - } - // Return a "storage trie" that is an adapter between the storge MPT - // and the unique verkle tree. - switch self := self.(type) { - case *trie.VerkleTrie: - return trie.NewTransitionTree(mpt.(*trie.StateTrie), self, true), nil - case *trie.TransitionTrie: - return trie.NewTransitionTree(mpt.(*trie.StateTrie), self.Overlay(), true), nil - default: - panic("unexpected trie type") - } - } - if db.started { - mpt, err := db.openStorageMPTrie(db.LastMerkleRoot, address, root, nil) - if err != nil { - return nil, err - } - // Return a "storage trie" that is an adapter between the storge MPT - // and the unique verkle tree. - switch self := self.(type) { - case *trie.VerkleTrie: - return trie.NewTransitionTree(mpt.(*trie.SecureTrie), self, true), nil - case *trie.TransitionTrie: - return trie.NewTransitionTree(mpt.(*trie.SecureTrie), self.Overlay(), true), nil - default: - panic("unexpected trie type") - } + return self, nil } mpt, err := db.openStorageMPTrie(stateRoot, address, root, nil) return mpt, err @@ -406,8 +338,6 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.StateTrie: return t.Copy() - case *trie.TransitionTrie: - return t.Copy() case *trie.VerkleTrie: return t.Copy() default: diff --git a/core/state/statedb.go b/core/state/statedb.go index 3b0135f9e9..1abdb7d475 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1051,14 +1051,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - root := s.trie.Hash() - - // Save the root of the MPT so that it can be used during the transition - if !s.Database().InTransition() && !s.Database().Transitioned() { - s.Database().SetLastMerkleRoot(root) - } - - return root + return s.trie.Hash() } // SetTxContext sets the current transaction hash and index which are diff --git a/core/state_processor.go b/core/state_processor.go index 4ccb06ecfb..fd76e53d8c 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,31 +17,18 @@ package core import ( - "bufio" - "bytes" - "encoding/binary" "errors" "fmt" - "io" "math/big" - "os" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - tutils "github.com/ethereum/go-ethereum/trie/utils" - "github.com/gballet/go-verkle" - "github.com/holiman/uint256" ) // StateProcessor is a basic Processor, which takes care of transitioning @@ -109,211 +96,9 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, errors.New("withdrawals before shanghai") } - // Overlay tree migration logic - migrdb := statedb.Database() - - // verkle transition: if the conversion process is in progress, move - // N values from the MPT into the verkle tree. - if migrdb.InTransition() { - var ( - now = time.Now() - tt = statedb.GetTrie().(*trie.TransitionTrie) - mpt = tt.Base() - vkt = tt.Overlay() - hasPreimagesBin = false - preimageSeek = migrdb.GetCurrentPreimageOffset() - fpreimages *bufio.Reader - ) - - // TODO: avoid opening the preimages file here and make it part of, potentially, statedb.Database(). - filePreimages, err := os.Open("preimages.bin") - if err != nil { - // fallback on reading the db - log.Warn("opening preimage file", "error", err) - } else { - defer filePreimages.Close() - if _, err := filePreimages.Seek(preimageSeek, io.SeekStart); err != nil { - return nil, nil, 0, fmt.Errorf("seeking preimage file: %s", err) - } - fpreimages = bufio.NewReader(filePreimages) - hasPreimagesBin = true - } - - accIt, err := statedb.Snaps().AccountIterator(mpt.Hash(), migrdb.GetCurrentAccountHash()) - if err != nil { - return nil, nil, 0, err - } - defer accIt.Release() - accIt.Next() - - // If we're about to start with the migration process, we have to read the first account hash preimage. - if migrdb.GetCurrentAccountAddress() == nil { - var addr common.Address - if hasPreimagesBin { - if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { - return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) - } - } else { - addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash())) - if len(addr) != 20 { - return nil, nil, 0, fmt.Errorf("addr len is zero is not 32: %d", len(addr)) - } - } - migrdb.SetCurrentAccountAddress(addr) - if migrdb.GetCurrentAccountHash() != accIt.Hash() { - return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) - } - preimageSeek += int64(len(addr)) - } - - const maxMovedCount = 10000 - // mkv will be assiting in the collection of up to maxMovedCount key values to be migrated to the VKT. - // It has internal caches to do efficient MPT->VKT key calculations, which will be discarded after - // this function. - mkv := &keyValueMigrator{vktLeafData: make(map[string]*verkle.BatchNewLeafNodeData)} - // move maxCount accounts into the verkle tree, starting with the - // slots from the previous account. - count := 0 - - // if less than maxCount slots were moved, move to the next account - for count < maxMovedCount { - acc, err := types.FullAccount(accIt.Account()) - if err != nil { - log.Error("Invalid account encountered during traversal", "error", err) - return nil, nil, 0, err - } - vkt.SetStorageRootConversion(*migrdb.GetCurrentAccountAddress(), acc.Root) - - // Start with processing the storage, because once the account is - // converted, the `stateRoot` field loses its meaning. Which means - // that it opens the door to a situation in which the storage isn't - // converted, but it can not be found since the account was and so - // there is no way to find the MPT storage from the information found - // in the verkle account. - // Note that this issue can still occur if the account gets written - // to during normal block execution. A mitigation strategy has been - // introduced with the `*StorageRootConversion` fields in VerkleDB. - if acc.HasStorage() { - stIt, err := statedb.Snaps().StorageIterator(mpt.Hash(), accIt.Hash(), migrdb.GetCurrentSlotHash()) - if err != nil { - return nil, nil, 0, err - } - stIt.Next() - - // fdb.StorageProcessed will be initialized to `true` if the - // entire storage for an account was not entirely processed - // by the previous block. This is used as a signal to resume - // processing the storage for that account where we left off. - // If the entire storage was processed, then the iterator was - // created in vain, but it's ok as this will not happen often. - for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ { - var ( - value []byte // slot value after RLP decoding - safeValue [32]byte // 32-byte aligned value - ) - if err := rlp.DecodeBytes(stIt.Slot(), &value); err != nil { - return nil, nil, 0, fmt.Errorf("error decoding bytes %x: %w", stIt.Slot(), err) - } - copy(safeValue[32-len(value):], value) - - var slotnr []byte - if hasPreimagesBin { - var s [32]byte - slotnr = s[:] - if _, err := io.ReadFull(fpreimages, slotnr); err != nil { - return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) - } - } else { - slotnr = rawdb.ReadPreimage(migrdb.DiskDB(), stIt.Hash()) - if len(slotnr) != 32 { - return nil, nil, 0, fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr)) - } - } - if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() { - return nil, nil, 0, fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr), stIt.Hash()) - } - preimageSeek += int64(len(slotnr)) - - mkv.addStorageSlot(migrdb.GetCurrentAccountAddress().Bytes(), slotnr, safeValue[:]) - - // advance the storage iterator - migrdb.SetStorageProcessed(!stIt.Next()) - if !migrdb.GetStorageProcessed() { - migrdb.SetCurrentSlotHash(stIt.Hash()) - } - } - stIt.Release() - } - - // If the maximum number of leaves hasn't been reached, then - // it means that the storage has finished processing (or none - // was available for this account) and that the account itself - // can be processed. - if count < maxMovedCount { - count++ // count increase for the account itself - - mkv.addAccount(migrdb.GetCurrentAccountAddress().Bytes(), acc) - vkt.ClearStrorageRootConversion(*migrdb.GetCurrentAccountAddress()) - - // Store the account code if present - if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) { - code := rawdb.ReadCode(statedb.Database().DiskDB(), common.BytesToHash(acc.CodeHash)) - chunks := trie.ChunkifyCode(code) - - mkv.addAccountCode(migrdb.GetCurrentAccountAddress().Bytes(), uint64(len(code)), chunks) - } - - // reset storage iterator marker for next account - migrdb.SetStorageProcessed(false) - migrdb.SetCurrentSlotHash(common.Hash{}) - - // Move to the next account, if available - or end - // the transition otherwise. - if accIt.Next() { - var addr common.Address - if hasPreimagesBin { - if _, err := io.ReadFull(fpreimages, addr[:]); err != nil { - return nil, nil, 0, fmt.Errorf("reading preimage file: %s", err) - } - } else { - addr = common.BytesToAddress(rawdb.ReadPreimage(migrdb.DiskDB(), accIt.Hash())) - if len(addr) != 20 { - return nil, nil, 0, fmt.Errorf("account address len is zero is not 20: %d", len(addr)) - } - } - // fmt.Printf("account switch: %s != %s\n", crypto.Keccak256Hash(addr[:]), accIt.Hash()) - if crypto.Keccak256Hash(addr[:]) != accIt.Hash() { - return nil, nil, 0, fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash()) - } - preimageSeek += int64(len(addr)) - migrdb.SetCurrentAccountAddress(addr) - } else { - // case when the account iterator has - // reached the end but count < maxCount - migrdb.EndVerkleTransition() - break - } - } - } - migrdb.SetCurrentPreimageOffset(preimageSeek) - - log.Info("Collected and prepared key values from base tree", "count", count, "duration", time.Since(now), "last account", statedb.Database().GetCurrentAccountHash()) - - now = time.Now() - if err := mkv.migrateCollectedKeyValues(tt.Overlay()); err != nil { - return nil, nil, 0, fmt.Errorf("could not migrate key values: %w", err) - } - log.Info("Inserted key values in overlay tree", "count", count, "duration", time.Since(now)) - } - // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) p.engine.Finalize(p.bc, header, statedb, block.Transactions(), block.Uncles(), withdrawals) - if block.NumberU64()%100 == 0 { - stateRoot := statedb.GetTrie().Hash() - log.Info("State root", "number", block.NumberU64(), "hash", stateRoot) - } - return receipts, allLogs, *usedGas, nil } @@ -379,117 +164,3 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo vmenv := vm.NewEVM(blockContext, vm.TxContext{BlobHashes: tx.BlobHashes()}, statedb, config, cfg) return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } - -// keyValueMigrator is a helper struct that collects key-values from the base tree. -// The walk is done in account order, so **we assume** the APIs hold this invariant. This is -// useful to be smart about caching banderwagon.Points to make VKT key calculations faster. -type keyValueMigrator struct { - currAddr []byte - currAddrPoint *verkle.Point - - vktLeafData map[string]*verkle.BatchNewLeafNodeData -} - -func (kvm *keyValueMigrator) addStorageSlot(addr []byte, slotNumber []byte, slotValue []byte) { - addrPoint := kvm.getAddrPoint(addr) - - vktKey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotNumber) - leafNodeData := kvm.getOrInitLeafNodeData(vktKey) - - leafNodeData.Values[vktKey[verkle.StemSize]] = slotValue -} - -func (kvm *keyValueMigrator) addAccount(addr []byte, acc *types.StateAccount) { - addrPoint := kvm.getAddrPoint(addr) - - vktKey := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint) - leafNodeData := kvm.getOrInitLeafNodeData(vktKey) - - var version [verkle.LeafValueSize]byte - leafNodeData.Values[tutils.VersionLeafKey] = version[:] - - var balance [verkle.LeafValueSize]byte - for i, b := range acc.Balance.Bytes() { - balance[len(acc.Balance.Bytes())-1-i] = b - } - leafNodeData.Values[tutils.BalanceLeafKey] = balance[:] - - var nonce [verkle.LeafValueSize]byte - binary.LittleEndian.PutUint64(nonce[:8], acc.Nonce) - leafNodeData.Values[tutils.NonceLeafKey] = nonce[:] - - leafNodeData.Values[tutils.CodeKeccakLeafKey] = acc.CodeHash[:] - - // Code size is ignored here. If this isn't an EOA, the tree-walk will call - // addAccountCode with this information. -} - -func (kvm *keyValueMigrator) addAccountCode(addr []byte, codeSize uint64, chunks []byte) { - addrPoint := kvm.getAddrPoint(addr) - - vktKey := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint) - leafNodeData := kvm.getOrInitLeafNodeData(vktKey) - - // Save the code size. - var codeSizeBytes [verkle.LeafValueSize]byte - binary.LittleEndian.PutUint64(codeSizeBytes[:8], codeSize) - leafNodeData.Values[tutils.CodeSizeLeafKey] = codeSizeBytes[:] - - // The first 128 chunks are stored in the account header leaf. - for i := 0; i < 128 && i < len(chunks)/32; i++ { - leafNodeData.Values[byte(128+i)] = chunks[32*i : 32*(i+1)] - } - - // Potential further chunks, have their own leaf nodes. - for i := 128; i < len(chunks)/32; { - vktKey := tutils.GetTreeKeyCodeChunkWithEvaluatedAddress(addrPoint, uint256.NewInt(uint64(i))) - leafNodeData := kvm.getOrInitLeafNodeData(vktKey) - - j := i - for ; (j-i) < 256 && j < len(chunks)/32; j++ { - leafNodeData.Values[byte((j-128)%256)] = chunks[32*j : 32*(j+1)] - } - i = j - } -} - -func (kvm *keyValueMigrator) getAddrPoint(addr []byte) *verkle.Point { - if bytes.Equal(addr, kvm.currAddr) { - return kvm.currAddrPoint - } - kvm.currAddr = addr - kvm.currAddrPoint = tutils.EvaluateAddressPoint(addr) - return kvm.currAddrPoint -} - -func (kvm *keyValueMigrator) getOrInitLeafNodeData(stem []byte) *verkle.BatchNewLeafNodeData { - stemStr := string(stem) - if _, ok := kvm.vktLeafData[stemStr]; !ok { - kvm.vktLeafData[stemStr] = &verkle.BatchNewLeafNodeData{ - Stem: stem[:verkle.StemSize], - Values: make(map[byte][]byte), - } - } - return kvm.vktLeafData[stemStr] -} - -func (kvm *keyValueMigrator) migrateCollectedKeyValues(tree *trie.VerkleTrie) error { - // Transform the map into a slice. - nodeValues := make([]verkle.BatchNewLeafNodeData, 0, len(kvm.vktLeafData)) - for _, vld := range kvm.vktLeafData { - nodeValues = append(nodeValues, *vld) - } - - // Create all leaves in batch mode so we can optimize cryptography operations. - newLeaves, err := verkle.BatchNewLeafNode(nodeValues) - if err != nil { - return fmt.Errorf("failed to batch-create new leaf nodes") - } - - // Insert into the tree. - if err := tree.InsertMigratedLeaves(newLeaves); err != nil { - return fmt.Errorf("failed to insert migrated leaves: %w", err) - } - - return nil -} diff --git a/light/trie.go b/light/trie.go index e0a283fdc1..0037a4073e 100644 --- a/light/trie.go +++ b/light/trie.go @@ -109,10 +109,6 @@ func (db *odrDatabase) EndVerkleTransition() { panic("not implemented") // TODO: Implement } -func (db *odrDatabase) InTransition() bool { - panic("not implemented") // TODO: Implement -} - func (db *odrDatabase) Transitioned() bool { panic("not implemented") // TODO: Implement } diff --git a/trie/database.go b/trie/database.go index 2c67232cc3..692e604dcf 100644 --- a/trie/database.go +++ b/trie/database.go @@ -236,40 +236,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return hdb.Node(hash) } -func (db *Database) HasStorageRootConversion(addr common.Address) bool { - db.addrToRootLock.RLock() - defer db.addrToRootLock.RUnlock() - if db.addrToRoot == nil { - return false - } - _, ok := db.addrToRoot[addr] - return ok -} - -func (db *Database) SetStorageRootConversion(addr common.Address, root common.Hash) { - db.addrToRootLock.Lock() - defer db.addrToRootLock.Unlock() - if db.addrToRoot == nil { - db.addrToRoot = make(map[common.Address]common.Hash) - } - db.addrToRoot[addr] = root -} - -func (db *Database) StorageRootConversion(addr common.Address) common.Hash { - db.addrToRootLock.RLock() - defer db.addrToRootLock.RUnlock() - if db.addrToRoot == nil { - return common.Hash{} - } - return db.addrToRoot[addr] -} - -func (db *Database) ClearStorageRootConversion(addr common.Address) { - db.addrToRootLock.Lock() - defer db.addrToRootLock.Unlock() - delete(db.addrToRoot, addr) -} - func (db *Database) IsVerkle() bool { return db.config != nil && db.config.Verkle } diff --git a/trie/transition.go b/trie/transition.go deleted file mode 100644 index ad5a7dc701..0000000000 --- a/trie/transition.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2021 go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/gballet/go-verkle" -) - -type TransitionTrie struct { - overlay *VerkleTrie - base *SecureTrie - storage bool -} - -func NewTransitionTree(base *SecureTrie, overlay *VerkleTrie, st bool) *TransitionTrie { - return &TransitionTrie{ - overlay: overlay, - base: base, - storage: st, - } -} - -func (t *TransitionTrie) Base() *SecureTrie { - return t.base -} - -// TODO(gballet/jsign): consider removing this API. -func (t *TransitionTrie) Overlay() *VerkleTrie { - return t.overlay -} - -// GetKey returns the sha3 preimage of a hashed key that was previously used -// to store a value. -// -// TODO(fjl): remove this when StateTrie is removed -func (t *TransitionTrie) GetKey(key []byte) []byte { - if key := t.overlay.GetKey(key); key != nil { - return key - } - return t.base.GetKey(key) -} - -// Get returns the value for key stored in the trie. The value bytes must -// not be modified by the caller. If a node was not found in the database, a -// trie.MissingNodeError is returned. -func (t *TransitionTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - if val, err := t.overlay.GetStorage(addr, key); len(val) != 0 || err != nil { - return val, nil - } - // TODO also insert value into overlay - return t.base.GetStorage(addr, key) -} - -// GetAccount abstract an account read from the trie. -func (t *TransitionTrie) GetAccount(address common.Address) (*types.StateAccount, error) { - data, err := t.overlay.GetAccount(address) - if err != nil { - // WORKAROUND, see the definition of errDeletedAccount - // for an explainer of why this if is needed. - if err == errDeletedAccount { - return nil, nil - } - return nil, err - } - if data != nil { - if t.overlay.db.HasStorageRootConversion(address) { - data.Root = t.overlay.db.StorageRootConversion(address) - } - return data, nil - } - // TODO also insert value into overlay - return t.base.GetAccount(address) -} - -// Update associates key with value in the trie. If value has length zero, any -// existing value is deleted from the trie. The value bytes must not be modified -// by the caller while they are stored in the trie. If a node was not found in the -// database, a trie.MissingNodeError is returned. -func (t *TransitionTrie) UpdateStorage(address common.Address, key []byte, value []byte) error { - var v []byte - if len(value) >= 32 { - v = value[:32] - } else { - var val [32]byte - copy(val[32-len(value):], value[:]) - v = val[:] - } - return t.overlay.UpdateStorage(address, key, v) -} - -// UpdateAccount abstract an account write to the trie. -func (t *TransitionTrie) UpdateAccount(addr common.Address, account *types.StateAccount) error { - if account.Root != (common.Hash{}) && account.Root != types.EmptyRootHash { - t.overlay.db.SetStorageRootConversion(addr, account.Root) - } - return t.overlay.UpdateAccount(addr, account) -} - -// Delete removes any existing value for key from the trie. If a node was not -// found in the database, a trie.MissingNodeError is returned. -func (t *TransitionTrie) DeleteStorage(addr common.Address, key []byte) error { - return t.overlay.DeleteStorage(addr, key) -} - -// DeleteAccount abstracts an account deletion from the trie. -func (t *TransitionTrie) DeleteAccount(key common.Address) error { - return t.overlay.DeleteAccount(key) -} - -// Hash returns the root hash of the trie. It does not write to the database and -// can be used even if the trie doesn't have one. -func (t *TransitionTrie) Hash() common.Hash { - return t.overlay.Hash() -} - -// Commit collects all dirty nodes in the trie and replace them with the -// corresponding node hash. All collected nodes(including dirty leaves if -// collectLeaf is true) will be encapsulated into a nodeset for return. -// The returned nodeset can be nil if the trie is clean(nothing to commit). -// Once the trie is committed, it's not usable anymore. A new trie must -// be created with new root and updated trie database for following usage -func (t *TransitionTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { - // Just return if the trie is a storage trie: otherwise, - // the overlay trie will be committed as many times as - // there are storage tries. This would kill performance. - if t.storage { - return common.Hash{}, nil, nil - } - return t.overlay.Commit(collectLeaf) -} - -// NodeIterator returns an iterator that returns nodes of the trie. Iteration -// starts at the key after the given start key. -func (t *TransitionTrie) NodeIterator(startKey []byte) (NodeIterator, error) { - panic("not implemented") // TODO: Implement -} - -// Prove constructs a Merkle proof for key. The result contains all encoded nodes -// on the path to the value at key. The value itself is also included in the last -// node and can be retrieved by verifying the proof. -// -// If the trie does not contain a value for key, the returned proof contains all -// nodes of the longest existing prefix of the key (at least the root), ending -// with the node that proves the absence of the key. -func (t *TransitionTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { - panic("not implemented") // TODO: Implement -} - -// IsVerkle returns true if the trie is verkle-tree based -func (t *TransitionTrie) IsVerkle() bool { - // For all intents and purposes, the calling code should treat this as a verkle trie - return true -} - -func (t *TransitionTrie) UpdateStem(key []byte, values [][]byte) error { - trie := t.overlay - switch root := trie.root.(type) { - case *verkle.InternalNode: - return root.InsertStem(key, values, t.overlay.flatdbNodeResolver) - default: - panic("invalid root type") - } -} - -func (t *TransitionTrie) Copy() *TransitionTrie { - return &TransitionTrie{ - overlay: t.overlay.Copy(), - base: t.base.Copy(), - storage: t.storage, - } -} - -func (t *TransitionTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { - return t.overlay.UpdateContractCode(addr, codeHash, code) -} diff --git a/trie/verkle.go b/trie/verkle.go index 385af5287e..b7a7d8f716 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -469,14 +469,6 @@ func ChunkifyCode(code []byte) ChunkedCode { return chunks } -func (t *VerkleTrie) SetStorageRootConversion(addr common.Address, root common.Hash) { - t.db.SetStorageRootConversion(addr, root) -} - -func (t *VerkleTrie) ClearStrorageRootConversion(addr common.Address) { - t.db.ClearStorageRootConversion(addr) -} - func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { var ( chunks = ChunkifyCode(code)