cmd/workload: rework tracegen to run tracing at block level (#32092)

This PR changes the trace test to block level, aiming for better
execution performance.

---------

Co-authored-by: zsfelfoldi <zsfelfoldi@gmail.com>

cmd/workload/README.md

@@ -17,6 +17,13 @@ and `eth_getBlockByNumber`, use this command:
 > ./workload test --sepolia --run History/getBlockBy http://host:8545
 ```
 
+Notably, trace tests require archive which keeps all the historical states for tracing.
+The additional flag is required to activate the trace tests.
+
+```
+> ./workload test --sepolia --archive --run Trace/Block http://host:8545
+```
+
 ### Regenerating tests
 
 There is a facility for updating the tests from the chain. This can also be used to
@@ -26,5 +33,5 @@ the following commands (in this directory) against a synced mainnet node:
 ```shell
 > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545
 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545
-> go run . tracegen --trace-tests queries/trace_mainnet.json http://host:8545
+> go run . tracegen --trace-tests queries/trace_mainnet.json --trace-start 4000000 --trace-end 4000100 http://host:8545
 ```

cmd/workload/filtertest.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math/big"
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
@@ -153,7 +154,14 @@ func (s *filterTestSuite) fullRangeQueryAndCheck(t *utesting.T, query *filterQue
 func (s *filterTestSuite) loadQueries() error {
 	file, err := s.cfg.fsys.Open(s.cfg.filterQueryFile)
 	if err != nil {
-		return fmt.Errorf("can't open filterQueryFile: %v", err)
+		// If not found in embedded FS, try to load it from disk
+		if !os.IsNotExist(err) {
+			return err
+		}
+		file, err = os.OpenFile(s.cfg.filterQueryFile, os.O_RDONLY, 0666)
+		if err != nil {
+			return fmt.Errorf("can't open filterQueryFile: %v", err)
+		}
 	}
 	defer file.Close()
 

cmd/workload/historytest.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -52,7 +53,14 @@ func newHistoryTestSuite(cfg testConfig) *historyTestSuite {
 func (s *historyTestSuite) loadTests() error {
 	file, err := s.cfg.fsys.Open(s.cfg.historyTestFile)
 	if err != nil {
-		return fmt.Errorf("can't open historyTestFile: %v", err)
+		// If not found in embedded FS, try to load it from disk
+		if !os.IsNotExist(err) {
+			return err
+		}
+		file, err = os.OpenFile(s.cfg.historyTestFile, os.O_RDONLY, 0666)
+		if err != nil {
+			return fmt.Errorf("can't open historyTestFile: %v", err)
+		}
 	}
 	defer file.Close()
 	if err := json.NewDecoder(file).Decode(&s.tests); err != nil {

cmd/workload/historytestgen.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -137,11 +138,17 @@ func calcReceiptsHash(rcpt []*types.Receipt) common.Hash {
 }
 
 func writeJSON(fileName string, value any) {
+	// Ensure the directory exists
+	dir := filepath.Dir(fileName)
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		exit(fmt.Errorf("failed to create directories: %w", err))
+	}
 	file, err := os.Create(fileName)
 	if err != nil {
 		exit(fmt.Errorf("error creating %s: %v", fileName, err))
 		return
 	}
 	defer file.Close()
+
 	json.NewEncoder(file).Encode(value)
 }

cmd/workload/testsuite.go

@@ -130,18 +130,42 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) {
 	switch {
 	case ctx.Bool(testMainnetFlag.Name):
 		cfg.fsys = builtinTestFiles
-		cfg.filterQueryFile = "queries/filter_queries_mainnet.json"
-		cfg.historyTestFile = "queries/history_mainnet.json"
+		if ctx.IsSet(filterQueryFileFlag.Name) {
+			cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)
+		} else {
+			cfg.filterQueryFile = "queries/filter_queries_mainnet.json"
+		}
+		if ctx.IsSet(historyTestFileFlag.Name) {
+			cfg.historyTestFile = ctx.String(historyTestFileFlag.Name)
+		} else {
+			cfg.historyTestFile = "queries/history_mainnet.json"
+		}
+		if ctx.IsSet(traceTestFileFlag.Name) {
+			cfg.traceTestFile = ctx.String(traceTestFileFlag.Name)
+		} else {
+			cfg.traceTestFile = "queries/trace_mainnet.json"
+		}
 		cfg.historyPruneBlock = new(uint64)
 		*cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber
-		cfg.traceTestFile = "queries/trace_mainnet.json"
 	case ctx.Bool(testSepoliaFlag.Name):
 		cfg.fsys = builtinTestFiles
-		cfg.filterQueryFile = "queries/filter_queries_sepolia.json"
-		cfg.historyTestFile = "queries/history_sepolia.json"
+		if ctx.IsSet(filterQueryFileFlag.Name) {
+			cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)
+		} else {
+			cfg.filterQueryFile = "queries/filter_queries_sepolia.json"
+		}
+		if ctx.IsSet(historyTestFileFlag.Name) {
+			cfg.historyTestFile = ctx.String(historyTestFileFlag.Name)
+		} else {
+			cfg.historyTestFile = "queries/history_sepolia.json"
+		}
+		if ctx.IsSet(traceTestFileFlag.Name) {
+			cfg.traceTestFile = ctx.String(traceTestFileFlag.Name)
+		} else {
+			cfg.traceTestFile = "queries/trace_sepolia.json"
+		}
 		cfg.historyPruneBlock = new(uint64)
 		*cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber
-		cfg.traceTestFile = "queries/trace_sepolia.json"
 	default:
 		cfg.fsys = os.DirFS(".")
 		cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)

cmd/workload/tracetest.go

@@ -33,7 +33,7 @@ import (
 
 // traceTest is the content of a history test.
 type traceTest struct {
-	TxHashes     []common.Hash         `json:"txHashes"`
+	BlockHashes  []common.Hash         `json:"blockHashes"`
 	TraceConfigs []tracers.TraceConfig `json:"traceConfigs"`
 	ResultHashes []common.Hash         `json:"resultHashes"`
 }
@@ -58,14 +58,20 @@ func newTraceTestSuite(cfg testConfig, ctx *cli.Context) *traceTestSuite {
 func (s *traceTestSuite) loadTests() error {
 	file, err := s.cfg.fsys.Open(s.cfg.traceTestFile)
 	if err != nil {
-		return fmt.Errorf("can't open traceTestFile: %v", err)
+		// If not found in embedded FS, try to load it from disk
+		if !os.IsNotExist(err) {
+			return err
+		}
+		file, err = os.OpenFile(s.cfg.traceTestFile, os.O_RDONLY, 0666)
+		if err != nil {
+			return fmt.Errorf("can't open traceTestFile: %v", err)
+		}
 	}
 	defer file.Close()
-
 	if err := json.NewDecoder(file).Decode(&s.tests); err != nil {
 		return fmt.Errorf("invalid JSON in %s: %v", s.cfg.traceTestFile, err)
 	}
-	if len(s.tests.TxHashes) == 0 {
+	if len(s.tests.BlockHashes) == 0 {
 		return fmt.Errorf("traceTestFile %s has no test data", s.cfg.traceTestFile)
 	}
 	return nil
@@ -73,17 +79,17 @@ func (s *traceTestSuite) loadTests() error {
 
 func (s *traceTestSuite) allTests() []workloadTest {
 	return []workloadTest{
-		newArchiveWorkloadTest("Trace/Transaction", s.traceTransaction),
+		newArchiveWorkloadTest("Trace/Block", s.traceBlock),
 	}
 }
 
-// traceTransaction runs all transaction tracing tests
-func (s *traceTestSuite) traceTransaction(t *utesting.T) {
+// traceBlock runs all block tracing tests
+func (s *traceTestSuite) traceBlock(t *utesting.T) {
 	ctx := context.Background()
 
-	for i, hash := range s.tests.TxHashes {
+	for i, hash := range s.tests.BlockHashes {
 		config := s.tests.TraceConfigs[i]
-		result, err := s.cfg.client.Geth.TraceTransaction(ctx, hash, &config)
+		result, err := s.cfg.client.Geth.TraceBlock(ctx, hash, &config)
 		if err != nil {
 			t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
 		}

cmd/workload/tracetestgen.go

@@ -36,8 +36,6 @@ import (
 )
 
 var (
-	defaultBlocksToTrace = 64 // the number of states assumed to be available
-
 	traceGenerateCommand = &cli.Command{
 		Name:      "tracegen",
 		Usage:     "Generates tests for state tracing",
@@ -46,7 +44,8 @@ var (
 		Flags: []cli.Flag{
 			traceTestFileFlag,
 			traceTestResultOutputFlag,
-			traceTestBlockFlag,
+			traceTestStartBlockFlag,
+			traceTestEndBlockFlag,
 		},
 	}
 
@@ -58,14 +57,18 @@ var (
 	}
 	traceTestResultOutputFlag = &cli.StringFlag{
 		Name:     "trace-output",
-		Usage:    "Folder containing the trace output files",
+		Usage:    "Folder containing detailed trace output files",
 		Value:    "",
 		Category: flags.TestingCategory,
 	}
-	traceTestBlockFlag = &cli.IntFlag{
-		Name:     "trace-blocks",
-		Usage:    "The number of blocks for tracing",
-		Value:    defaultBlocksToTrace,
+	traceTestStartBlockFlag = &cli.IntFlag{
+		Name:     "trace-start",
+		Usage:    "The number of starting block for tracing (included)",
+		Category: flags.TestingCategory,
+	}
+	traceTestEndBlockFlag = &cli.IntFlag{
+		Name:     "trace-end",
+		Usage:    "The number of ending block for tracing (excluded)",
 		Category: flags.TestingCategory,
 	}
 	traceTestInvalidOutputFlag = &cli.StringFlag{
@@ -81,21 +84,22 @@ func generateTraceTests(clictx *cli.Context) error {
 		client     = makeClient(clictx)
 		outputFile = clictx.String(traceTestFileFlag.Name)
 		outputDir  = clictx.String(traceTestResultOutputFlag.Name)
-		blocks     = clictx.Int(traceTestBlockFlag.Name)
+		startBlock = clictx.Int(traceTestStartBlockFlag.Name)
+		endBlock   = clictx.Int(traceTestEndBlockFlag.Name)
 		ctx        = context.Background()
 		test       = new(traceTest)
 	)
-	if outputDir != "" {
-		err := os.MkdirAll(outputDir, os.ModePerm)
-		if err != nil {
-			return err
-		}
-	}
 	latest, err := client.Eth.BlockNumber(ctx)
 	if err != nil {
 		exit(err)
 	}
-	if latest < uint64(blocks) {
+	if startBlock > endBlock {
+		exit(fmt.Errorf("invalid block range for tracing, start: %d, end: %d", startBlock, endBlock))
+	}
+	if endBlock-startBlock == 0 {
+		exit(fmt.Errorf("invalid block range for tracing, start: %d, end: %d", startBlock, endBlock))
+	}
+	if latest < uint64(startBlock) || latest < uint64(endBlock) {
 		exit(fmt.Errorf("node seems not synced, latest block is %d", latest))
 	}
 	// Get blocks and assign block info into the test
@@ -104,37 +108,35 @@ func generateTraceTests(clictx *cli.Context) error {
 		logged = time.Now()
 		failed int
 	)
-	log.Info("Trace transactions around the chain tip", "head", latest, "blocks", blocks)
+	log.Info("Trace transactions around the chain tip", "head", latest, "start", startBlock, "end", endBlock)
 
-	for i := 0; i < blocks; i++ {
-		number := latest - uint64(i)
-		block, err := client.Eth.BlockByNumber(ctx, big.NewInt(int64(number)))
+	for i := startBlock; i < endBlock; i++ {
+		header, err := client.Eth.HeaderByNumber(ctx, big.NewInt(int64(i)))
 		if err != nil {
 			exit(err)
 		}
-		for _, tx := range block.Transactions() {
-			config, configName := randomTraceOption()
-			result, err := client.Geth.TraceTransaction(ctx, tx.Hash(), config)
-			if err != nil {
-				failed += 1
-				break
-			}
-			blob, err := json.Marshal(result)
-			if err != nil {
-				failed += 1
-				break
-			}
-			test.TxHashes = append(test.TxHashes, tx.Hash())
-			test.TraceConfigs = append(test.TraceConfigs, *config)
-			test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob))
-			writeTraceResult(outputDir, tx.Hash(), result, configName)
+		config, configName := randomTraceOption()
+		result, err := client.Geth.TraceBlock(ctx, header.Hash(), config)
+		if err != nil {
+			failed += 1
+			continue
 		}
+		blob, err := json.Marshal(result)
+		if err != nil {
+			failed += 1
+			continue
+		}
+		test.BlockHashes = append(test.BlockHashes, header.Hash())
+		test.TraceConfigs = append(test.TraceConfigs, *config)
+		test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob))
+		writeTraceResult(outputDir, header.Hash(), result, configName)
+
 		if time.Since(logged) > time.Second*8 {
 			logged = time.Now()
-			log.Info("Tracing transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+			log.Info("Tracing blocks", "executed", len(test.BlockHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
 		}
 	}
-	log.Info("Traced transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Info("Traced blocks", "executed", len(test.BlockHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
 
 	// Write output file.
 	writeJSON(outputFile, test)
@@ -142,24 +144,15 @@ func generateTraceTests(clictx *cli.Context) error {
 }
 
 func randomTraceOption() (*tracers.TraceConfig, string) {
-	x := rand.Intn(11)
+	x := rand.Intn(10)
 	if x == 0 {
-		// struct-logger, with all fields enabled, very heavy
-		return &tracers.TraceConfig{
-			Config: &logger.Config{
-				EnableMemory:     true,
-				EnableReturnData: true,
-			},
-		}, "structAll"
-	}
-	if x == 1 {
 		// default options for struct-logger, with stack and storage capture
 		// enabled
 		return &tracers.TraceConfig{
 			Config: &logger.Config{},
 		}, "structDefault"
 	}
-	if x == 2 || x == 3 || x == 4 {
+	if x >= 1 && x <= 3 {
 		// struct-logger with storage capture enabled
 		return &tracers.TraceConfig{
 			Config: &logger.Config{
@@ -170,14 +163,18 @@ func randomTraceOption() (*tracers.TraceConfig, string) {
 	// Native tracer
 	loggers := []string{"callTracer", "4byteTracer", "flatCallTracer", "muxTracer", "noopTracer", "prestateTracer"}
 	return &tracers.TraceConfig{
-		Tracer: &loggers[x-5],
-	}, loggers[x-5]
+		Tracer: &loggers[x-4],
+	}, loggers[x-4]
 }
 
 func writeTraceResult(dir string, hash common.Hash, result any, configName string) {
 	if dir == "" {
 		return
 	}
+	// Ensure the directory exists
+	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+		exit(fmt.Errorf("failed to create directories: %w", err))
+	}
 	name := filepath.Join(dir, configName+"_"+hash.String())
 	file, err := os.Create(name)
 	if err != nil {

ethclient/gethclient/gethclient.go

@@ -216,6 +216,17 @@ func (ec *Client) TraceTransaction(ctx context.Context, hash common.Hash, config
 	return result, nil
 }
 
+// TraceBlock returns the structured logs created during the execution of EVM
+// and returns them as a JSON object.
+func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *tracers.TraceConfig) (any, error) {
+	var result any
+	err := ec.c.CallContext(ctx, &result, "debug_traceBlockByHash", hash, config)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
 func toBlockNumArg(number *big.Int) string {
 	if number == nil {
 		return "latest"