cmd/evm/t8ntool: stream alloc input into StateDB

For file-based alloc input, tokenize the JSON and insert each account
into the pre-state StateDB as it is decoded, instead of materializing
the full GenesisAlloc map first and then iterating it in MakePreState.
Peak memory becomes a single decoded account rather than the whole
alloc map, the decoder's buffer over it, and the StateDB populating
alongside both.

On the same benchmark-scale test that measured the output-streaming
change, this dropped t8n peak heap_sys from ~8 GiB to ~3 GiB and cut
an additional ~20s off wall-clock. The stdin input path still uses
the map-based MakePreState since alloc shares a decoder with env/txs
there; Prestate.AllocPath is how the file path opts in.

cmd/evm/internal/t8ntool/execution.go

@@ -17,9 +17,11 @@
 package t8ntool
 
 import (
+	"encoding/json"
 	"fmt"
 	stdmath "math"
 	"math/big"
+	"os"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -47,6 +49,9 @@ type Prestate struct {
 	Env        stEnv                         `json:"env"`
 	Pre        types.GenesisAlloc            `json:"pre"`
 	TreeLeaves map[common.Hash]hexutil.Bytes `json:"vkt,omitempty"`
+	// AllocPath, when non-empty, causes Apply to stream the alloc from disk
+	// instead of reading Pre, so the full map never materializes in memory.
+	AllocPath string `json:"-"`
 }
 
 //go:generate go run github.com/fjl/gencodec -type ExecutionResult -field-override executionResultMarshaling -out gen_execresult.go
@@ -146,8 +151,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		return h
 	}
 	var (
-		isEIP4762   = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp)
+		isEIP4762 = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp)
-		statedb     = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762)
+		statedb   *state.StateDB
+	)
+	if pre.AllocPath != "" {
+		var err error
+		statedb, err = MakePreStateStreaming(rawdb.NewMemoryDatabase(), pre.AllocPath, isEIP4762)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+	} else {
+		statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762)
+	}
+	var (
 		signer      = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp)
 		gaspool     = core.NewGasPool(pre.Env.GasLimit)
 		blockHash   = common.Hash{0x13, 0x37}
@@ -414,6 +430,76 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool
 	return statedb
 }
 
+// MakePreStateStreaming is like MakePreState, but decodes the alloc from disk
+// one account at a time so the full map is never held in memory.
+func MakePreStateStreaming(db ethdb.Database, allocPath string, isBintrie bool) (*state.StateDB, error) {
+	tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsUBT: isBintrie})
+	sdb := state.NewDatabase(tdb, nil)
+
+	root := types.EmptyRootHash
+	if isBintrie {
+		root = types.EmptyBinaryHash
+	}
+	statedb, err := state.New(root, sdb)
+	if err != nil {
+		return nil, NewError(ErrorEVM, fmt.Errorf("failed to create initial statedb: %v", err))
+	}
+
+	f, err := os.Open(allocPath)
+	if err != nil {
+		return nil, NewError(ErrorIO, fmt.Errorf("failed reading alloc file: %v", err))
+	}
+	defer f.Close()
+
+	dec := json.NewDecoder(f)
+	tok, err := dec.Token()
+	if err != nil {
+		return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc opening token: %v", err))
+	}
+	if d, ok := tok.(json.Delim); !ok || d != '{' {
+		return nil, NewError(ErrorJson, fmt.Errorf("expected alloc object, got %v", tok))
+	}
+	for dec.More() {
+		keyTok, err := dec.Token()
+		if err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc key: %v", err))
+		}
+		keyStr, ok := keyTok.(string)
+		if !ok {
+			return nil, NewError(ErrorJson, fmt.Errorf("alloc key not a string: %v", keyTok))
+		}
+		addr := common.HexToAddress(keyStr)
+		var acct types.Account
+		if err := dec.Decode(&acct); err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("failed decoding account %s: %v", keyStr, err))
+		}
+		statedb.SetCode(addr, acct.Code, tracing.CodeChangeUnspecified)
+		statedb.SetNonce(addr, acct.Nonce, tracing.NonceChangeGenesis)
+		if acct.Balance != nil {
+			statedb.SetBalance(addr, uint256.MustFromBig(acct.Balance), tracing.BalanceIncreaseGenesisBalance)
+		}
+		for k, v := range acct.Storage {
+			statedb.SetState(addr, k, v)
+		}
+	}
+	if _, err := dec.Token(); err != nil {
+		return nil, NewError(ErrorJson, fmt.Errorf("failed reading alloc closing token: %v", err))
+	}
+
+	root, err = statedb.Commit(0, false, false)
+	if err != nil {
+		return nil, NewError(ErrorEVM, fmt.Errorf("failed to commit initial state: %v", err))
+	}
+	if isBintrie {
+		return statedb, nil
+	}
+	statedb, err = state.New(root, sdb)
+	if err != nil {
+		return nil, NewError(ErrorEVM, fmt.Errorf("failed to reopen state after commit: %v", err))
+	}
+	return statedb, nil
+}
+
 func rlpHash(x any) (h common.Hash) {
 	hw := keccak.NewLegacyKeccak256()
 	rlp.Encode(hw, x)

cmd/evm/internal/t8ntool/transition.go

@@ -17,6 +17,7 @@
 package t8ntool
 
 import (
+	"bufio"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -115,11 +116,10 @@ func Transition(ctx *cli.Context) error {
 		}
 	}
 	if allocStr != stdinSelector {
-		if err := readFile(allocStr, "alloc", &inputData.Alloc); err != nil {
+		prestate.AllocPath = allocStr
-			return err
+	} else {
-		}
+		prestate.Pre = inputData.Alloc
 	}
-	prestate.Pre = inputData.Alloc
 
 	if btStr != stdinSelector && btStr != "" {
 		if err := readFile(btStr, "BT", &inputData.BT); err != nil {
@@ -224,24 +224,21 @@ func Transition(ctx *cli.Context) error {
 		}
 	}
 	// Dump the execution result.
-	//
-	// When the alloc output targets a regular file (not stdout/stderr or the
-	// binary-trie path), stream entries to disk to keep peak memory bounded
-	// to one account's JSON encoding rather than the full post-state.
 	var (
 		collector Alloc
 		btleaves  map[common.Hash]hexutil.Bytes
 	)
 	isBinary := chainConfig.IsUBT(big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp)
-	outputAlloc := ctx.String(OutputAllocFlag.Name)
+	allocOutput := ctx.String(OutputAllocFlag.Name)
-	streamAllocToDisk := !isBinary && outputAlloc != "" &&
-		outputAlloc != "stdout" && outputAlloc != "stderr"
-
 	switch {
-	case streamAllocToDisk:
+	case !isBinary && allocOutput != "" && allocOutput != "stdout" && allocOutput != "stderr":
-		if err := writeStreamedAlloc(filepath.Join(baseDir, outputAlloc), s); err != nil {
+		// Stream directly to the output file to avoid materializing the
+		// whole post-state in memory. dispatchOutput is told to skip alloc
+		// by clearing the output name.
+		if err := writeStreamedAlloc(filepath.Join(baseDir, allocOutput), s); err != nil {
 			return err
 		}
+		allocOutput = ""
 	case !isBinary:
 		collector = make(Alloc)
 		s.DumpToCollector(collector, nil)
@@ -251,23 +248,27 @@ func Transition(ctx *cli.Context) error {
 			return err
 		}
 	}
-	return dispatchOutput(ctx, baseDir, result, collector, body, btleaves, streamAllocToDisk)
+	return dispatchOutput(ctx, baseDir, result, collector, allocOutput, body, btleaves)
 }
 
-// writeStreamedAlloc writes the post-state alloc to path by streaming entries
+// writeStreamedAlloc writes the post-state alloc to path one account at a
-// from the state iterator, producing the same JSON shape as saveFile for an
+// time, producing the same JSON shape as saveFile on an Alloc map.
-// Alloc map without buffering the full state in memory.
 func writeStreamedAlloc(path string, s *state.StateDB) error {
 	f, err := os.Create(path)
 	if err != nil {
 		return NewError(ErrorIO, fmt.Errorf("failed creating alloc output file: %v", err))
 	}
-	sa := newStreamingAlloc(f)
+	bw := bufio.NewWriter(f)
+	sa := newStreamingAlloc(bw)
 	s.DumpToCollector(sa, nil)
 	if err := sa.Close(); err != nil {
 		f.Close()
 		return NewError(ErrorIO, fmt.Errorf("failed writing alloc output: %v", err))
 	}
+	if err := bw.Flush(); err != nil {
+		f.Close()
+		return NewError(ErrorIO, fmt.Errorf("failed flushing alloc output: %v", err))
+	}
 	if err := f.Close(); err != nil {
 		return NewError(ErrorIO, fmt.Errorf("failed closing alloc output file: %v", err))
 	}
@@ -364,8 +365,6 @@ func (g Alloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) {
 	g[*addr] = dumpAccountToTypesAccount(dumpAccount)
 }
 
-// dumpAccountToTypesAccount converts a state.DumpAccount into the types.Account
-// shape used for alloc output by both the buffered and streaming collectors.
 func dumpAccountToTypesAccount(dumpAccount state.DumpAccount) types.Account {
 	balance, _ := new(big.Int).SetString(dumpAccount.Balance, 0)
 	var storage map[common.Hash]common.Hash
@@ -383,12 +382,9 @@ func dumpAccountToTypesAccount(dumpAccount state.DumpAccount) types.Account {
 	}
 }
 
-// streamingAlloc is a DumpCollector that writes each account directly to an
+// streamingAlloc is a DumpCollector that writes each account to w as it is
-// io.Writer as a JSON object ({addr: account, ...}), producing the same
+// visited, emitting a single JSON object keyed by address. Close must be
-// shape as saveFile on an Alloc map. Peak memory is one account's marshal
+// called to emit the closing brace.
-// output rather than the whole post-state plus its MarshalIndent buffer.
-//
-// Usage: OnRoot (opening brace), OnAccount per entry, Close (closing brace).
 type streamingAlloc struct {
 	w        io.Writer
 	wroteOne bool
@@ -399,11 +395,15 @@ func newStreamingAlloc(w io.Writer) *streamingAlloc {
 	return &streamingAlloc{w: w}
 }
 
-func (s *streamingAlloc) OnRoot(common.Hash) {
+func (s *streamingAlloc) write(b []byte) {
 	if s.err != nil {
 		return
 	}
-	_, s.err = io.WriteString(s.w, "{")
+	_, s.err = s.w.Write(b)
+}
+
+func (s *streamingAlloc) OnRoot(common.Hash) {
+	s.write([]byte{'{'})
 }
 
 func (s *streamingAlloc) OnAccount(addr *common.Address, dumpAccount state.DumpAccount) {
@@ -420,24 +420,18 @@ func (s *streamingAlloc) OnAccount(addr *common.Address, dumpAccount state.DumpA
 		s.err = err
 		return
 	}
-	separator := ""
 	if s.wroteOne {
-		separator = ","
+		s.write([]byte{','})
-	}
-	if _, err := fmt.Fprintf(s.w, "%s%s:%s", separator, keyJSON, valueJSON); err != nil {
-		s.err = err
-		return
 	}
+	s.write(keyJSON)
+	s.write([]byte{':'})
+	s.write(valueJSON)
 	s.wroteOne = true
 }
 
-// Close finishes the JSON object by emitting the closing brace.
 func (s *streamingAlloc) Close() error {
-	if s.err != nil {
+	s.write([]byte{'}'})
-		return s.err
+	return s.err
-	}
-	_, err := io.WriteString(s.w, "}")
-	return err
 }
 
 // saveFile marshals the object to the given file
@@ -455,9 +449,9 @@ func saveFile(baseDir, filename string, data interface{}) error {
 }
 
 // dispatchOutput writes the output data to either stderr or stdout, or to the specified
-// files. When allocAlreadyWritten is true, the alloc has been streamed to disk
+// files. An empty allocOutput skips the alloc dispatch, which is used when the
-// by the caller and dispatchOutput skips its alloc branch.
+// alloc has already been streamed to disk by the caller.
-func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes, allocAlreadyWritten bool) error {
+func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, alloc Alloc, allocOutput string, body hexutil.Bytes, bt map[common.Hash]hexutil.Bytes) error {
 	stdOutObject := make(map[string]interface{})
 	stdErrObject := make(map[string]interface{})
 	dispatch := func(baseDir, fName, name string, obj interface{}) error {
@@ -475,10 +469,8 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a
 		}
 		return nil
 	}
-	if !allocAlreadyWritten {
+	if err := dispatch(baseDir, allocOutput, "alloc", alloc); err != nil {
-		if err := dispatch(baseDir, ctx.String(OutputAllocFlag.Name), "alloc", alloc); err != nil {
+		return err
-			return err
-		}
 	}
 	if err := dispatch(baseDir, ctx.String(OutputResultFlag.Name), "result", result); err != nil {
 		return err