diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index f17829ec53..253ebe1111 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -17,9 +17,11 @@ package t8ntool import ( + "encoding/json" "fmt" stdmath "math" "math/big" + "os" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -47,6 +49,9 @@ type Prestate struct { Env stEnv `json:"env"` Pre types.GenesisAlloc `json:"pre"` TreeLeaves map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"` + // AllocPath, when non-empty, causes Apply to stream the alloc from disk + // instead of reading Pre, so the full map never materializes in memory. + AllocPath string `json:"-"` } //go:generate go run github.com/fjl/gencodec -type ExecutionResult -field-override executionResultMarshaling -out gen_execresult.go @@ -146,8 +151,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return h } var ( - isEIP4762 = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) - statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) + isEIP4762 = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) + statedb *state.StateDB + ) + if pre.AllocPath != "" { + var err error + statedb, err = MakePreStateStreaming(rawdb.NewMemoryDatabase(), pre.AllocPath, isEIP4762) + if err != nil { + return nil, nil, nil, err + } + } else { + statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) + } + var ( signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) gaspool = core.NewGasPool(pre.Env.GasLimit) blockHash = common.Hash{0x13, 0x37} @@ -414,6 +430,76 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool return statedb } +// MakePreStateStreaming is like MakePreState, but decodes the alloc from disk +// one account at a time so the full map is never held in memory. +func MakePreStateStreaming(db ethdb.Database, allocPath string, isBintrie bool) (*state.StateDB, error) { + tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsUBT: isBintrie}) + sdb := state.NewDatabase(tdb, nil) + + root := types.EmptyRootHash + if isBintrie { + root = types.EmptyBinaryHash + } + statedb, err := state.New(root, sdb) + if err != nil { + return nil, NewError(ErrorEVM, fmt.Errorf("failed to create initial statedb: %v", err)) + } + + f, err := os.Open(allocPath) + if err != nil { + return nil, NewError(ErrorIO, fmt.Errorf("failed reading alloc file: %v", err)) + } + defer f.Close() + + dec := json.NewDecoder(f) + tok, err := dec.Token() + if err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc opening token: %v", err)) + } + if d, ok := tok.(json.Delim); !ok || d != '{' { + return nil, NewError(ErrorJson, fmt.Errorf("expected alloc object, got %v", tok)) + } + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc key: %v", err)) + } + keyStr, ok := keyTok.(string) + if !ok { + return nil, NewError(ErrorJson, fmt.Errorf("alloc key not a string: %v", keyTok)) + } + addr := common.HexToAddress(keyStr) + var acct types.Account + if err := dec.Decode(&acct); err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("failed decoding account %s: %v", keyStr, err)) + } + statedb.SetCode(addr, acct.Code, tracing.CodeChangeUnspecified) + statedb.SetNonce(addr, acct.Nonce, tracing.NonceChangeGenesis) + if acct.Balance != nil { + statedb.SetBalance(addr, uint256.MustFromBig(acct.Balance), tracing.BalanceIncreaseGenesisBalance) + } + for k, v := range acct.Storage { + statedb.SetState(addr, k, v) + } + } + if _, err := dec.Token(); err != nil { + return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc closing token: %v", err)) + } + + root, err = statedb.Commit(0, false, false) + if err != nil { + return nil, NewError(ErrorEVM, fmt.Errorf("failed to commit initial state: %v", err)) + } + if isBintrie { + return statedb, nil + } + statedb, err = state.New(root, sdb) + if err != nil { + return nil, NewError(ErrorEVM, fmt.Errorf("failed to reopen state after commit: %v", err)) + } + return statedb, nil +} + func rlpHash(x any) (h common.Hash) { hw := keccak.NewLegacyKeccak256() rlp.Encode(hw, x) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 580ebfe5b4..e0bb3a449d 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -17,6 +17,7 @@ package t8ntool import ( + "bufio" "encoding/json" "errors" "fmt" @@ -115,11 +116,10 @@ func Transition(ctx *cli.Context) error { } } if allocStr != stdinSelector { - if err := readFile(allocStr, "alloc", &inputData.Alloc); err != nil { - return err - } + prestate.AllocPath = allocStr + } else { + prestate.Pre = inputData.Alloc } - prestate.Pre = inputData.Alloc if btStr != stdinSelector && btStr != "" { if err := readFile(btStr, "BT", &inputData.BT); err != nil { @@ -224,24 +224,21 @@ func Transition(ctx *cli.Context) error { } } // Dump the execution result. - // - // When the alloc output targets a regular file (not stdout/stderr or the - // binary-trie path), stream entries to disk to keep peak memory bounded - // to one account's JSON encoding rather than the full post-state. var ( collector Alloc btleaves map[common.Hash]hexutil.Bytes ) isBinary := chainConfig.IsUBT(big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp) - outputAlloc := ctx.String(OutputAllocFlag.Name) - streamAllocToDisk := !isBinary && outputAlloc != "" && - outputAlloc != "stdout" && outputAlloc != "stderr" - + allocOutput := ctx.String(OutputAllocFlag.Name) switch { - case streamAllocToDisk: - if err := writeStreamedAlloc(filepath.Join(baseDir, outputAlloc), s); err != nil { + case !isBinary && allocOutput != "" && allocOutput != "stdout" && allocOutput != "stderr": + // Stream directly to the output file to avoid materializing the + // whole post-state in memory. dispatchOutput is told to skip alloc + // by clearing the output name. + if err := writeStreamedAlloc(filepath.Join(baseDir, allocOutput), s); err != nil { return err } + allocOutput = "" case !isBinary: collector = make(Alloc) s.DumpToCollector(collector, nil) @@ -251,23 +248,27 @@ func Transition(ctx *cli.Context) error { return err } } - return dispatchOutput(ctx, baseDir, result, collector, body, btleaves, streamAllocToDisk) + return dispatchOutput(ctx, baseDir, result, collector, allocOutput, body, btleaves) } -// writeStreamedAlloc writes the post-state alloc to path by streaming entries -// from the state iterator, producing the same JSON shape as saveFile for an -// Alloc map without buffering the full state in memory. +// writeStreamedAlloc writes the post-state alloc to path one account at a +// time, producing the same JSON shape as saveFile on an Alloc map. func writeStreamedAlloc(path string, s *state.StateDB) error { f, err := os.Create(path) if err != nil { return NewError(ErrorIO, fmt.Errorf("failed creating alloc output file: %v", err)) } - sa := newStreamingAlloc(f) + bw := bufio.NewWriter(f) + sa := newStreamingAlloc(bw) s.DumpToCollector(sa, nil) if err := sa.Close(); err != nil { f.Close() return NewError(ErrorIO, fmt.Errorf("failed writing alloc output: %v", err)) } + if err := bw.Flush(); err != nil { + f.Close() + return NewError(ErrorIO, fmt.Errorf("failed flushing alloc output: %v", err)) + } if err := f.Close(); err != nil { return NewError(ErrorIO, fmt.Errorf("failed closing alloc output file: %v", err)) } @@ -364,8 +365,6 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { g[*addr] = dumpAccountToTypesAccount(dumpAccount) } -// dumpAccountToTypesAccount converts a state.DumpAccount into the types.Account -// shape used for alloc output by both the buffered and streaming collectors. func dumpAccountToTypesAccount(dumpAccount state.DumpAccount) types.Account { balance, _ := new(big.Int).SetString(dumpAccount.Balance, 0) var storage map[common.Hash]common.Hash @@ -383,12 +382,9 @@ func dumpAccountToTypesAccount(dumpAccount state.DumpAccount) types.Account { } } -// streamingAlloc is a DumpCollector that writes each account directly to an -// io.Writer as a JSON object ({addr: account, ...}), producing the same -// shape as saveFile on an Alloc map. Peak memory is one account's marshal -// output rather than the whole post-state plus its MarshalIndent buffer. -// -// Usage: OnRoot (opening brace), OnAccount per entry, Close (closing brace). +// streamingAlloc is a DumpCollector that writes each account to w as it is +// visited, emitting a single JSON object keyed by address. Close must be +// called to emit the closing brace. type streamingAlloc struct { w io.Writer wroteOne bool @@ -399,11 +395,15 @@ func newStreamingAlloc(w io.Writer) *streamingAlloc { return &streamingAlloc{w: w} } -func (s *streamingAlloc) OnRoot(common.Hash) { +func (s *streamingAlloc) write(b []byte) { if s.err != nil { return } - _, s.err = io.WriteString(s.w, "{") + _, s.err = s.w.Write(b) +} + +func (s *streamingAlloc) OnRoot(common.Hash) { + s.write([]byte{'{'}) } func (s *streamingAlloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) { @@ -420,24 +420,18 @@ func (s *streamingAlloc) OnAccount(addr *common.Address, dumpAccount state.DumpA s.err = err return } - separator := "" if s.wroteOne { - separator = "," - } - if _, err := fmt.Fprintf(s.w, "%s%s:%s", separator, keyJSON, valueJSON); err != nil { - s.err = err - return + s.write([]byte{','}) } + s.write(keyJSON) + s.write([]byte{':'}) + s.write(valueJSON) s.wroteOne = true } -// Close finishes the JSON object by emitting the closing brace. func (s *streamingAlloc) Close() error { - if s.err != nil { - return s.err - } - _, err := io.WriteString(s.w, "}") - return err + s.write([]byte{'}'}) + return s.err } // saveFile marshals the object to the given file @@ -455,9 +449,9 @@ func saveFile(baseDir, filename string, data interface{}) error { } // dispatchOutput writes the output data to either stderr or stdout, or to the specified -// files. When allocAlreadyWritten is true, the alloc has been streamed to disk -// by the caller and dispatchOutput skips its alloc branch. -func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes, allocAlreadyWritten bool) error { +// files. An empty allocOutput skips the alloc dispatch, which is used when the +// alloc has already been streamed to disk by the caller. +func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, allocOutput string, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes) error { stdOutObject := make(map[string]interface{}) stdErrObject := make(map[string]interface{}) dispatch := func(baseDir, fName, name string, obj interface{}) error { @@ -475,10 +469,8 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a } return nil } - if !allocAlreadyWritten { - if err := dispatch(baseDir, ctx.String(OutputAllocFlag.Name), "alloc", alloc); err != nil { - return err - } + if err := dispatch(baseDir, allocOutput, "alloc", alloc); err != nil { + return err } if err := dispatch(baseDir, ctx.String(OutputResultFlag.Name), "result", result); err != nil { return err