From 79d0d4cf79e23caf0765d3ea05da0f9388665bfa Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 17 Jun 2025 16:27:56 +0800 Subject: [PATCH] cmd/workload: rework tracegen to run tracing at block level --- cmd/workload/README.md | 9 +++- cmd/workload/filtertest.go | 11 +++- cmd/workload/historytest.go | 11 +++- cmd/workload/historytestgen.go | 7 +++ cmd/workload/testsuite.go | 36 ++++++++++--- cmd/workload/tracetest.go | 30 +++++++---- cmd/workload/tracetestgen.go | 81 ++++++++++++++++-------------- ethclient/gethclient/gethclient.go | 11 ++++ 8 files changed, 139 insertions(+), 57 deletions(-) diff --git a/cmd/workload/README.md b/cmd/workload/README.md index bf1978eebd..1b84dd05db 100644 --- a/cmd/workload/README.md +++ b/cmd/workload/README.md @@ -17,6 +17,13 @@ and `eth_getBlockByNumber`, use this command: > ./workload test --sepolia --run History/getBlockBy http://host:8545 ``` +Notably, trace tests require archive which keeps all the historical states for tracing. +The additional flag is required to activate the trace tests. + +``` +> ./workload test --sepolia --archive --run Trace/Block http://host:8545 +``` + ### Regenerating tests There is a facility for updating the tests from the chain. This can also be used to @@ -26,5 +33,5 @@ the following commands (in this directory) against a synced mainnet node: ```shell > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545 -> go run . tracegen --trace-tests queries/trace_mainnet.json http://host:8545 +> go run . tracegen --trace-tests queries/trace_mainnet.json --trace-start 4000000 --trace-end 4000100 http://host:8545 ``` diff --git a/cmd/workload/filtertest.go b/cmd/workload/filtertest.go index 68b8891852..174d13222b 100644 --- a/cmd/workload/filtertest.go +++ b/cmd/workload/filtertest.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math/big" + "os" "time" "github.com/ethereum/go-ethereum" @@ -153,7 +154,15 @@ func (s *filterTestSuite) fullRangeQueryAndCheck(t *utesting.T, query *filterQue func (s *filterTestSuite) loadQueries() error { file, err := s.cfg.fsys.Open(s.cfg.filterQueryFile) if err != nil { - return fmt.Errorf("can't open filterQueryFile: %v", err) + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.filterQueryFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open filterQueryFile: %v", err) + } + defer file.Close() } defer file.Close() diff --git a/cmd/workload/historytest.go b/cmd/workload/historytest.go index bfe8989f24..198ba945e0 100644 --- a/cmd/workload/historytest.go +++ b/cmd/workload/historytest.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "os" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -52,7 +53,15 @@ func newHistoryTestSuite(cfg testConfig) *historyTestSuite { func (s *historyTestSuite) loadTests() error { file, err := s.cfg.fsys.Open(s.cfg.historyTestFile) if err != nil { - return fmt.Errorf("can't open historyTestFile: %v", err) + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.historyTestFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open historyTestFile: %v", err) + } + defer file.Close() } defer file.Close() if err := json.NewDecoder(file).Decode(&s.tests); err != nil { diff --git a/cmd/workload/historytestgen.go b/cmd/workload/historytestgen.go index 002945f2a6..b88c8208da 100644 --- a/cmd/workload/historytestgen.go +++ b/cmd/workload/historytestgen.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "os" + "path/filepath" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -137,11 +138,17 @@ func calcReceiptsHash(rcpt []*types.Receipt) common.Hash { } func writeJSON(fileName string, value any) { + // Ensure the directory exists + dir := filepath.Dir(fileName) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + exit(fmt.Errorf("failed to create directories: %w", err)) + } file, err := os.Create(fileName) if err != nil { exit(fmt.Errorf("error creating %s: %v", fileName, err)) return } defer file.Close() + json.NewEncoder(file).Encode(value) } diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index 2aa9b0cb11..39eeb8e3c2 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -130,18 +130,42 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { switch { case ctx.Bool(testMainnetFlag.Name): cfg.fsys = builtinTestFiles - cfg.filterQueryFile = "queries/filter_queries_mainnet.json" - cfg.historyTestFile = "queries/history_mainnet.json" + if ctx.IsSet(filterQueryFileFlag.Name) { + cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) + } else { + cfg.filterQueryFile = "queries/filter_queries_mainnet.json" + } + if ctx.IsSet(historyTestFileFlag.Name) { + cfg.historyTestFile = ctx.String(historyTestFileFlag.Name) + } else { + cfg.historyTestFile = "queries/history_mainnet.json" + } + if ctx.IsSet(traceTestFileFlag.Name) { + cfg.traceTestFile = ctx.String(traceTestFileFlag.Name) + } else { + cfg.traceTestFile = "queries/trace_mainnet.json" + } cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber - cfg.traceTestFile = "queries/trace_mainnet.json" case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles - cfg.filterQueryFile = "queries/filter_queries_sepolia.json" - cfg.historyTestFile = "queries/history_sepolia.json" + if ctx.IsSet(filterQueryFileFlag.Name) { + cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) + } else { + cfg.filterQueryFile = "queries/filter_queries_sepolia.json" + } + if ctx.IsSet(historyTestFileFlag.Name) { + cfg.historyTestFile = ctx.String(historyTestFileFlag.Name) + } else { + cfg.historyTestFile = "queries/history_sepolia.json" + } + if ctx.IsSet(traceTestFileFlag.Name) { + cfg.traceTestFile = ctx.String(traceTestFileFlag.Name) + } else { + cfg.traceTestFile = "queries/trace_sepolia.json" + } cfg.historyPruneBlock = new(uint64) *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber - cfg.traceTestFile = "queries/trace_sepolia.json" default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) diff --git a/cmd/workload/tracetest.go b/cmd/workload/tracetest.go index 2cb818dfb7..837f13fee0 100644 --- a/cmd/workload/tracetest.go +++ b/cmd/workload/tracetest.go @@ -33,7 +33,7 @@ import ( // traceTest is the content of a history test. type traceTest struct { - TxHashes []common.Hash `json:"txHashes"` + BlockHashes []common.Hash `json:"blockHashes"` TraceConfigs []tracers.TraceConfig `json:"traceConfigs"` ResultHashes []common.Hash `json:"resultHashes"` } @@ -57,15 +57,23 @@ func newTraceTestSuite(cfg testConfig, ctx *cli.Context) *traceTestSuite { func (s *traceTestSuite) loadTests() error { file, err := s.cfg.fsys.Open(s.cfg.traceTestFile) - if err != nil { - return fmt.Errorf("can't open traceTestFile: %v", err) + if err == nil { + defer file.Close() + } else { + // If not found in embedded FS, try to load it from disk + if !os.IsNotExist(err) { + return err + } + file, err = os.OpenFile(s.cfg.traceTestFile, os.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("can't open traceTestFile: %v", err) + } + defer file.Close() } - defer file.Close() - if err := json.NewDecoder(file).Decode(&s.tests); err != nil { return fmt.Errorf("invalid JSON in %s: %v", s.cfg.traceTestFile, err) } - if len(s.tests.TxHashes) == 0 { + if len(s.tests.BlockHashes) == 0 { return fmt.Errorf("traceTestFile %s has no test data", s.cfg.traceTestFile) } return nil @@ -73,17 +81,17 @@ func (s *traceTestSuite) loadTests() error { func (s *traceTestSuite) allTests() []workloadTest { return []workloadTest{ - newArchiveWorkloadTest("Trace/Transaction", s.traceTransaction), + newArchiveWorkloadTest("Trace/Block", s.traceBlock), } } -// traceTransaction runs all transaction tracing tests -func (s *traceTestSuite) traceTransaction(t *utesting.T) { +// traceBlock runs all block tracing tests +func (s *traceTestSuite) traceBlock(t *utesting.T) { ctx := context.Background() - for i, hash := range s.tests.TxHashes { + for i, hash := range s.tests.BlockHashes { config := s.tests.TraceConfigs[i] - result, err := s.cfg.client.Geth.TraceTransaction(ctx, hash, &config) + result, err := s.cfg.client.Geth.TraceBlock(ctx, hash, &config) if err != nil { t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err) } diff --git a/cmd/workload/tracetestgen.go b/cmd/workload/tracetestgen.go index db36b5bb20..d796412ad3 100644 --- a/cmd/workload/tracetestgen.go +++ b/cmd/workload/tracetestgen.go @@ -36,8 +36,6 @@ import ( ) var ( - defaultBlocksToTrace = 64 // the number of states assumed to be available - traceGenerateCommand = &cli.Command{ Name: "tracegen", Usage: "Generates tests for state tracing", @@ -46,7 +44,8 @@ var ( Flags: []cli.Flag{ traceTestFileFlag, traceTestResultOutputFlag, - traceTestBlockFlag, + traceTestStartBlockFlag, + traceTestEndBlockFlag, }, } @@ -58,14 +57,18 @@ var ( } traceTestResultOutputFlag = &cli.StringFlag{ Name: "trace-output", - Usage: "Folder containing the trace output files", + Usage: "Folder containing detailed trace output files", Value: "", Category: flags.TestingCategory, } - traceTestBlockFlag = &cli.IntFlag{ - Name: "trace-blocks", - Usage: "The number of blocks for tracing", - Value: defaultBlocksToTrace, + traceTestStartBlockFlag = &cli.IntFlag{ + Name: "trace-start", + Usage: "The number of starting block for tracing (included)", + Category: flags.TestingCategory, + } + traceTestEndBlockFlag = &cli.IntFlag{ + Name: "trace-end", + Usage: "The number of ending block for tracing (excluded)", Category: flags.TestingCategory, } traceTestInvalidOutputFlag = &cli.StringFlag{ @@ -81,21 +84,22 @@ func generateTraceTests(clictx *cli.Context) error { client = makeClient(clictx) outputFile = clictx.String(traceTestFileFlag.Name) outputDir = clictx.String(traceTestResultOutputFlag.Name) - blocks = clictx.Int(traceTestBlockFlag.Name) + startBlock = clictx.Int(traceTestStartBlockFlag.Name) + endBlock = clictx.Int(traceTestEndBlockFlag.Name) ctx = context.Background() test = new(traceTest) ) - if outputDir != "" { - err := os.MkdirAll(outputDir, os.ModePerm) - if err != nil { - return err - } - } latest, err := client.Eth.BlockNumber(ctx) if err != nil { exit(err) } - if latest < uint64(blocks) { + if startBlock > endBlock { + exit(fmt.Errorf("invalid block range for tracing, start: %d, end: %d", startBlock, endBlock)) + } + if endBlock-startBlock == 0 { + exit(fmt.Errorf("invalid block range for tracing, start: %d, end: %d", startBlock, endBlock)) + } + if latest < uint64(startBlock) || latest < uint64(endBlock) { exit(fmt.Errorf("node seems not synced, latest block is %d", latest)) } // Get blocks and assign block info into the test @@ -104,37 +108,36 @@ func generateTraceTests(clictx *cli.Context) error { logged = time.Now() failed int ) - log.Info("Trace transactions around the chain tip", "head", latest, "blocks", blocks) + log.Info("Trace transactions around the chain tip", "head", latest, "start", startBlock, "end", endBlock) - for i := 0; i < blocks; i++ { + for i := startBlock; i < endBlock; i++ { number := latest - uint64(i) - block, err := client.Eth.BlockByNumber(ctx, big.NewInt(int64(number))) + header, err := client.Eth.HeaderByNumber(ctx, big.NewInt(int64(number))) if err != nil { exit(err) } - for _, tx := range block.Transactions() { - config, configName := randomTraceOption() - result, err := client.Geth.TraceTransaction(ctx, tx.Hash(), config) - if err != nil { - failed += 1 - break - } - blob, err := json.Marshal(result) - if err != nil { - failed += 1 - break - } - test.TxHashes = append(test.TxHashes, tx.Hash()) - test.TraceConfigs = append(test.TraceConfigs, *config) - test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob)) - writeTraceResult(outputDir, tx.Hash(), result, configName) + config, configName := randomTraceOption() + result, err := client.Geth.TraceBlock(ctx, header.Hash(), config) + if err != nil { + failed += 1 + continue } + blob, err := json.Marshal(result) + if err != nil { + failed += 1 + continue + } + test.BlockHashes = append(test.BlockHashes, header.Hash()) + test.TraceConfigs = append(test.TraceConfigs, *config) + test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob)) + writeTraceResult(outputDir, header.Hash(), result, configName) + if time.Since(logged) > time.Second*8 { logged = time.Now() - log.Info("Tracing transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Tracing blocks", "executed", len(test.BlockHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start))) } } - log.Info("Traced transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Traced blocks", "executed", len(test.BlockHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start))) // Write output file. writeJSON(outputFile, test) @@ -178,6 +181,10 @@ func writeTraceResult(dir string, hash common.Hash, result any, configName strin if dir == "" { return } + // Ensure the directory exists + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + exit(fmt.Errorf("failed to create directories: %w", err)) + } name := filepath.Join(dir, configName+"_"+hash.String()) file, err := os.Create(name) if err != nil { diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 2ea2ebbc3e..d030878e54 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -216,6 +216,17 @@ func (ec *Client) TraceTransaction(ctx context.Context, hash common.Hash, config return result, nil } +// TraceBlock returns the structured logs created during the execution of EVM +// and returns them as a JSON object. +func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *tracers.TraceConfig) (any, error) { + var result any + err := ec.c.CallContext(ctx, &result, "debug_traceBlockByHash", hash, config) + if err != nil { + return nil, err + } + return result, nil +} + func toBlockNumArg(number *big.Int) string { if number == nil { return "latest"