cmd/workload: introduce transaction-trace test (#31288)

This pull request introduces a new test suite in workload framework, for
transaction tracing.

**test generation**
`go run . tracegen --trace-tests trace-test.json http://host:8545`

and you can choose to store the trace result in a specific folder
`go run . tracegen --trace-tests trace-test.json --trace-output
./trace-result http://host:8545`

**test run**
`./workload test -run Trace/Transaction --trace-invalid ./trace-invalid
http://host:8545`

The mismatched trace result will be saved in the specific folder for
further investigation.

cmd/workload/README.md

@@ -26,4 +26,5 @@ the following commands (in this directory) against a synced mainnet node:
 ```shell
 > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545
 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545
+> go run . tracegen --trace-tests queries/trace_mainnet.json http://host:8545
 ```

cmd/workload/client.go

@@ -0,0 +1,104 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/ethclient/gethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/urfave/cli/v2"
+)
+
+type client struct {
+	Eth  *ethclient.Client
+	Geth *gethclient.Client
+	RPC  *rpc.Client
+}
+
+func makeClient(ctx *cli.Context) *client {
+	if ctx.NArg() < 1 {
+		exit("missing RPC endpoint URL as command-line argument")
+	}
+	url := ctx.Args().First()
+	cl, err := rpc.Dial(url)
+	if err != nil {
+		exit(fmt.Errorf("could not create RPC client at %s: %v", url, err))
+	}
+	return &client{
+		RPC:  cl,
+		Eth:  ethclient.NewClient(cl),
+		Geth: gethclient.New(cl),
+	}
+}
+
+type simpleBlock struct {
+	Number hexutil.Uint64 `json:"number"`
+	Hash   common.Hash    `json:"hash"`
+}
+
+type simpleTransaction struct {
+	Hash             common.Hash    `json:"hash"`
+	TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
+}
+
+func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
+	var r *simpleBlock
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
+	return r, err
+}
+
+func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
+	var r *simpleBlock
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
+	return r, err
+}
+
+func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
+	var r *simpleTransaction
+	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
+	return r, err
+}
+
+func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
+	var r *simpleTransaction
+	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
+	return r, err
+}
+
+func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
+	var r hexutil.Uint64
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
+	return uint64(r), err
+}
+
+func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
+	var r hexutil.Uint64
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
+	return uint64(r), err
+}
+
+func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
+	var result []*types.Receipt
+	err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
+	return result, err
+}

cmd/workload/filtertest.go

@@ -45,11 +45,11 @@ func newFilterTestSuite(cfg testConfig) *filterTestSuite {
 	return s
 }
 
-func (s *filterTestSuite) allTests() []utesting.Test {
+func (s *filterTestSuite) allTests() []workloadTest {
-	return []utesting.Test{
+	return []workloadTest{
-		{Name: "Filter/ShortRange", Fn: s.filterShortRange},
+		newWorkLoadTest("Filter/ShortRange", s.filterShortRange),
-		{Name: "Filter/LongRange", Fn: s.filterLongRange, Slow: true},
+		newSlowWorkloadTest("Filter/LongRange", s.filterLongRange),
-		{Name: "Filter/FullRange", Fn: s.filterFullRange, Slow: true},
+		newSlowWorkloadTest("Filter/FullRange", s.filterFullRange),
 	}
 }
 

cmd/workload/historytest.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/utesting"
 )
 
@@ -65,40 +64,16 @@ func (s *historyTestSuite) loadTests() error {
 	return nil
 }
 
-func (s *historyTestSuite) allTests() []utesting.Test {
+func (s *historyTestSuite) allTests() []workloadTest {
-	return []utesting.Test{
+	return []workloadTest{
-		{
+		newWorkLoadTest("History/getBlockByHash", s.testGetBlockByHash),
-			Name: "History/getBlockByHash",
+		newWorkLoadTest("History/getBlockByNumber", s.testGetBlockByNumber),
-			Fn:   s.testGetBlockByHash,
+		newWorkLoadTest("History/getBlockReceiptsByHash", s.testGetBlockReceiptsByHash),
-		},
+		newWorkLoadTest("History/getBlockReceiptsByNumber", s.testGetBlockReceiptsByNumber),
-		{
+		newWorkLoadTest("History/getBlockTransactionCountByHash", s.testGetBlockTransactionCountByHash),
-			Name: "History/getBlockByNumber",
+		newWorkLoadTest("History/getBlockTransactionCountByNumber", s.testGetBlockTransactionCountByNumber),
-			Fn:   s.testGetBlockByNumber,
+		newWorkLoadTest("History/getTransactionByBlockHashAndIndex", s.testGetTransactionByBlockHashAndIndex),
-		},
+		newWorkLoadTest("History/getTransactionByBlockNumberAndIndex", s.testGetTransactionByBlockNumberAndIndex),
-		{
-			Name: "History/getBlockReceiptsByHash",
-			Fn:   s.testGetBlockReceiptsByHash,
-		},
-		{
-			Name: "History/getBlockReceiptsByNumber",
-			Fn:   s.testGetBlockReceiptsByNumber,
-		},
-		{
-			Name: "History/getBlockTransactionCountByHash",
-			Fn:   s.testGetBlockTransactionCountByHash,
-		},
-		{
-			Name: "History/getBlockTransactionCountByNumber",
-			Fn:   s.testGetBlockTransactionCountByNumber,
-		},
-		{
-			Name: "History/getTransactionByBlockHashAndIndex",
-			Fn:   s.testGetTransactionByBlockHashAndIndex,
-		},
-		{
-			Name: "History/getTransactionByBlockNumberAndIndex",
-			Fn:   s.testGetTransactionByBlockNumberAndIndex,
-		},
 	}
 }
 
@@ -279,55 +254,3 @@ func (s *historyTestSuite) testGetTransactionByBlockNumberAndIndex(t *utesting.T
 		}
 	}
 }
-
-type simpleBlock struct {
-	Number hexutil.Uint64 `json:"number"`
-	Hash   common.Hash    `json:"hash"`
-}
-
-type simpleTransaction struct {
-	Hash             common.Hash    `json:"hash"`
-	TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
-}
-
-func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
-	var r *simpleBlock
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
-	return r, err
-}
-
-func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
-	var r *simpleBlock
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
-	return r, err
-}
-
-func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
-	var r *simpleTransaction
-	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
-	return r, err
-}
-
-func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
-	var r *simpleTransaction
-	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
-	return r, err
-}
-
-func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
-	var r hexutil.Uint64
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
-	return uint64(r), err
-}
-
-func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
-	var r hexutil.Uint64
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
-	return uint64(r), err
-}
-
-func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
-	var result []*types.Receipt
-	err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
-	return result, err
-}

cmd/workload/historytestgen.go

@@ -51,7 +51,7 @@ var (
 	}
 	historyTestEarliestFlag = &cli.IntFlag{
 		Name:     "earliest",
-		Usage:    "JSON file containing filter test queries",
+		Usage:    "The earliest block to test queries",
 		Value:    0,
 		Category: flags.TestingCategory,
 	}
@@ -139,7 +139,7 @@ func calcReceiptsHash(rcpt []*types.Receipt) common.Hash {
 func writeJSON(fileName string, value any) {
 	file, err := os.Create(fileName)
 	if err != nil {
-		exit(fmt.Errorf("Error creating %s: %v", fileName, err))
+		exit(fmt.Errorf("error creating %s: %v", fileName, err))
 		return
 	}
 	defer file.Close()

cmd/workload/main.go

@@ -20,10 +20,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/urfave/cli/v2"
 )
 
@@ -49,6 +47,7 @@ func init() {
 		runTestCommand,
 		historyGenerateCommand,
 		filterGenerateCommand,
+		traceGenerateCommand,
 		filterPerfCommand,
 	}
 }
@@ -57,26 +56,6 @@ func main() {
 	exit(app.Run(os.Args))
 }
 
-type client struct {
-	Eth *ethclient.Client
-	RPC *rpc.Client
-}
-
-func makeClient(ctx *cli.Context) *client {
-	if ctx.NArg() < 1 {
-		exit("missing RPC endpoint URL as command-line argument")
-	}
-	url := ctx.Args().First()
-	cl, err := rpc.Dial(url)
-	if err != nil {
-		exit(fmt.Errorf("Could not create RPC client at %s: %v", url, err))
-	}
-	return &client{
-		RPC: cl,
-		Eth: ethclient.NewClient(cl),
-	}
-}
-
 func exit(err any) {
 	if err == nil {
 		os.Exit(0)

cmd/workload/testsuite.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
-	"slices"
 
 	"github.com/ethereum/go-ethereum/core/history"
 	"github.com/ethereum/go-ethereum/internal/flags"
@@ -45,10 +44,13 @@ var (
 			testPatternFlag,
 			testTAPFlag,
 			testSlowFlag,
+			testArchiveFlag,
 			testSepoliaFlag,
 			testMainnetFlag,
 			filterQueryFileFlag,
 			historyTestFileFlag,
+			traceTestFileFlag,
+			traceTestInvalidOutputFlag,
 		},
 	}
 	testPatternFlag = &cli.StringFlag{
@@ -67,6 +69,12 @@ var (
 		Value:    false,
 		Category: flags.TestingCategory,
 	}
+	testArchiveFlag = &cli.BoolFlag{
+		Name:     "archive",
+		Usage:    "Enable archive tests",
+		Value:    false,
+		Category: flags.TestingCategory,
+	}
 	testSepoliaFlag = &cli.BoolFlag{
 		Name:     "sepolia",
 		Usage:    "Use test cases for sepolia network",
@@ -86,6 +94,7 @@ type testConfig struct {
 	filterQueryFile   string
 	historyTestFile   string
 	historyPruneBlock *uint64
+	traceTestFile     string
 }
 
 var errPrunedHistory = fmt.Errorf("attempt to access pruned history")
@@ -125,36 +134,85 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) {
 		cfg.historyTestFile = "queries/history_mainnet.json"
 		cfg.historyPruneBlock = new(uint64)
 		*cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber
+		cfg.traceTestFile = "queries/trace_mainnet.json"
 	case ctx.Bool(testSepoliaFlag.Name):
 		cfg.fsys = builtinTestFiles
 		cfg.filterQueryFile = "queries/filter_queries_sepolia.json"
 		cfg.historyTestFile = "queries/history_sepolia.json"
 		cfg.historyPruneBlock = new(uint64)
 		*cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber
+		cfg.traceTestFile = "queries/trace_sepolia.json"
 	default:
 		cfg.fsys = os.DirFS(".")
 		cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)
 		cfg.historyTestFile = ctx.String(historyTestFileFlag.Name)
+		cfg.traceTestFile = ctx.String(traceTestFileFlag.Name)
 	}
 	return cfg
 }
 
+// workloadTest represents a single test in the workload. It's a wrapper
+// of utesting.Test by adding a few additional attributes.
+type workloadTest struct {
+	utesting.Test
+
+	archive bool // Flag whether the archive node (full state history) is required for this test
+}
+
+func newWorkLoadTest(name string, fn func(t *utesting.T)) workloadTest {
+	return workloadTest{
+		Test: utesting.Test{
+			Name: name,
+			Fn:   fn,
+		},
+	}
+}
+
+func newSlowWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
+	t := newWorkLoadTest(name, fn)
+	t.Slow = true
+	return t
+}
+
+func newArchiveWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
+	t := newWorkLoadTest(name, fn)
+	t.archive = true
+	return t
+}
+
+func filterTests(tests []workloadTest, pattern string, filterFn func(t workloadTest) bool) []utesting.Test {
+	var utests []utesting.Test
+	for _, t := range tests {
+		if filterFn(t) {
+			utests = append(utests, t.Test)
+		}
+	}
+	if pattern == "" {
+		return utests
+	}
+	return utesting.MatchTests(utests, pattern)
+}
+
 func runTestCmd(ctx *cli.Context) error {
 	cfg := testConfigFromCLI(ctx)
 	filterSuite := newFilterTestSuite(cfg)
 	historySuite := newHistoryTestSuite(cfg)
+	traceSuite := newTraceTestSuite(cfg, ctx)
 
 	// Filter test cases.
 	tests := filterSuite.allTests()
 	tests = append(tests, historySuite.allTests()...)
-	if ctx.IsSet(testPatternFlag.Name) {
+	tests = append(tests, traceSuite.allTests()...)
-		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
+
-	}
+	utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool {
-	if !ctx.Bool(testSlowFlag.Name) {
+		if t.Slow && !ctx.Bool(testSlowFlag.Name) {
-		tests = slices.DeleteFunc(tests, func(test utesting.Test) bool {
+			return false
-			return test.Slow
+		}
-		})
+		if t.archive && !ctx.Bool(testArchiveFlag.Name) {
-	}
+			return false
+		}
+		return true
+	})
 
 	// Disable logging unless explicitly enabled.
 	if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") {
@@ -166,7 +224,7 @@ func runTestCmd(ctx *cli.Context) error {
 	if ctx.Bool(testTAPFlag.Name) {
 		run = utesting.RunTAP
 	}
-	results := run(tests, os.Stdout)
+	results := run(utests, os.Stdout)
 	if utesting.CountFailures(results) > 0 {
 		os.Exit(1)
 	}

cmd/workload/tracetest.go

@@ -0,0 +1,126 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+// traceTest is the content of a history test.
+type traceTest struct {
+	TxHashes     []common.Hash         `json:"txHashes"`
+	TraceConfigs []tracers.TraceConfig `json:"traceConfigs"`
+	ResultHashes []common.Hash         `json:"resultHashes"`
+}
+
+type traceTestSuite struct {
+	cfg        testConfig
+	tests      traceTest
+	invalidDir string
+}
+
+func newTraceTestSuite(cfg testConfig, ctx *cli.Context) *traceTestSuite {
+	s := &traceTestSuite{
+		cfg:        cfg,
+		invalidDir: ctx.String(traceTestInvalidOutputFlag.Name),
+	}
+	if err := s.loadTests(); err != nil {
+		exit(err)
+	}
+	return s
+}
+
+func (s *traceTestSuite) loadTests() error {
+	file, err := s.cfg.fsys.Open(s.cfg.traceTestFile)
+	if err != nil {
+		return fmt.Errorf("can't open traceTestFile: %v", err)
+	}
+	defer file.Close()
+
+	if err := json.NewDecoder(file).Decode(&s.tests); err != nil {
+		return fmt.Errorf("invalid JSON in %s: %v", s.cfg.traceTestFile, err)
+	}
+	if len(s.tests.TxHashes) == 0 {
+		return fmt.Errorf("traceTestFile %s has no test data", s.cfg.traceTestFile)
+	}
+	return nil
+}
+
+func (s *traceTestSuite) allTests() []workloadTest {
+	return []workloadTest{
+		newArchiveWorkloadTest("Trace/Transaction", s.traceTransaction),
+	}
+}
+
+// traceTransaction runs all transaction tracing tests
+func (s *traceTestSuite) traceTransaction(t *utesting.T) {
+	ctx := context.Background()
+
+	for i, hash := range s.tests.TxHashes {
+		config := s.tests.TraceConfigs[i]
+		result, err := s.cfg.client.Geth.TraceTransaction(ctx, hash, &config)
+		if err != nil {
+			t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
+		}
+		blob, err := json.Marshal(result)
+		if err != nil {
+			t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
+			continue
+		}
+		if crypto.Keccak256Hash(blob) != s.tests.ResultHashes[i] {
+			t.Errorf("Transaction %d (hash %v): invalid result", i, hash)
+
+			writeInvalidTraceResult(s.invalidDir, hash, result)
+		}
+	}
+}
+
+func writeInvalidTraceResult(dir string, hash common.Hash, result any) {
+	if dir == "" {
+		return
+	}
+	err := os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		log.Info("Failed to make output directory", "err", err)
+		return
+	}
+	name := filepath.Join(dir, "invalid"+"_"+hash.String())
+	file, err := os.Create(name)
+	if err != nil {
+		exit(fmt.Errorf("error creating %s: %v", name, err))
+		return
+	}
+	defer file.Close()
+
+	data, _ := json.MarshalIndent(result, "", "    ")
+	_, err = file.Write(data)
+	if err != nil {
+		exit(fmt.Errorf("error writing %s: %v", name, err))
+		return
+	}
+}

cmd/workload/tracetestgen.go

@@ -0,0 +1,195 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+	"github.com/ethereum/go-ethereum/eth/tracers/logger"
+	"github.com/ethereum/go-ethereum/internal/flags"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	defaultBlocksToTrace = 64 // the number of states assumed to be available
+
+	traceGenerateCommand = &cli.Command{
+		Name:      "tracegen",
+		Usage:     "Generates tests for state tracing",
+		ArgsUsage: "<RPC endpoint URL>",
+		Action:    generateTraceTests,
+		Flags: []cli.Flag{
+			traceTestFileFlag,
+			traceTestResultOutputFlag,
+			traceTestBlockFlag,
+		},
+	}
+
+	traceTestFileFlag = &cli.StringFlag{
+		Name:     "trace-tests",
+		Usage:    "JSON file containing trace test queries",
+		Value:    "trace_tests.json",
+		Category: flags.TestingCategory,
+	}
+	traceTestResultOutputFlag = &cli.StringFlag{
+		Name:     "trace-output",
+		Usage:    "Folder containing the trace output files",
+		Value:    "",
+		Category: flags.TestingCategory,
+	}
+	traceTestBlockFlag = &cli.IntFlag{
+		Name:     "trace-blocks",
+		Usage:    "The number of blocks for tracing",
+		Value:    defaultBlocksToTrace,
+		Category: flags.TestingCategory,
+	}
+	traceTestInvalidOutputFlag = &cli.StringFlag{
+		Name:     "trace-invalid",
+		Usage:    "Folder containing the mismatched trace output files",
+		Value:    "",
+		Category: flags.TestingCategory,
+	}
+)
+
+func generateTraceTests(clictx *cli.Context) error {
+	var (
+		client     = makeClient(clictx)
+		outputFile = clictx.String(traceTestFileFlag.Name)
+		outputDir  = clictx.String(traceTestResultOutputFlag.Name)
+		blocks     = clictx.Int(traceTestBlockFlag.Name)
+		ctx        = context.Background()
+		test       = new(traceTest)
+	)
+	if outputDir != "" {
+		err := os.MkdirAll(outputDir, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+	latest, err := client.Eth.BlockNumber(ctx)
+	if err != nil {
+		exit(err)
+	}
+	if latest < uint64(blocks) {
+		exit(fmt.Errorf("node seems not synced, latest block is %d", latest))
+	}
+	// Get blocks and assign block info into the test
+	var (
+		start  = time.Now()
+		logged = time.Now()
+		failed int
+	)
+	log.Info("Trace transactions around the chain tip", "head", latest, "blocks", blocks)
+
+	for i := 0; i < blocks; i++ {
+		number := latest - uint64(i)
+		block, err := client.Eth.BlockByNumber(ctx, big.NewInt(int64(number)))
+		if err != nil {
+			exit(err)
+		}
+		for _, tx := range block.Transactions() {
+			config, configName := randomTraceOption()
+			result, err := client.Geth.TraceTransaction(ctx, tx.Hash(), config)
+			if err != nil {
+				failed += 1
+				break
+			}
+			blob, err := json.Marshal(result)
+			if err != nil {
+				failed += 1
+				break
+			}
+			test.TxHashes = append(test.TxHashes, tx.Hash())
+			test.TraceConfigs = append(test.TraceConfigs, *config)
+			test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob))
+			writeTraceResult(outputDir, tx.Hash(), result, configName)
+		}
+		if time.Since(logged) > time.Second*8 {
+			logged = time.Now()
+			log.Info("Tracing transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+		}
+	}
+	log.Info("Traced transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+
+	// Write output file.
+	writeJSON(outputFile, test)
+	return nil
+}
+
+func randomTraceOption() (*tracers.TraceConfig, string) {
+	x := rand.Intn(11)
+	if x == 0 {
+		// struct-logger, with all fields enabled, very heavy
+		return &tracers.TraceConfig{
+			Config: &logger.Config{
+				EnableMemory:     true,
+				EnableReturnData: true,
+			},
+		}, "structAll"
+	}
+	if x == 1 {
+		// default options for struct-logger, with stack and storage capture
+		// enabled
+		return &tracers.TraceConfig{
+			Config: &logger.Config{},
+		}, "structDefault"
+	}
+	if x == 2 || x == 3 || x == 4 {
+		// struct-logger with storage capture enabled
+		return &tracers.TraceConfig{
+			Config: &logger.Config{
+				DisableStack: true,
+			},
+		}, "structStorage"
+	}
+	// Native tracer
+	loggers := []string{"callTracer", "4byteTracer", "flatCallTracer", "muxTracer", "noopTracer", "prestateTracer"}
+	return &tracers.TraceConfig{
+		Tracer: &loggers[x-5],
+	}, loggers[x-5]
+}
+
+func writeTraceResult(dir string, hash common.Hash, result any, configName string) {
+	if dir == "" {
+		return
+	}
+	name := filepath.Join(dir, configName+"_"+hash.String())
+	file, err := os.Create(name)
+	if err != nil {
+		exit(fmt.Errorf("error creating %s: %v", name, err))
+		return
+	}
+	defer file.Close()
+
+	data, _ := json.MarshalIndent(result, "", "    ")
+	_, err = file.Write(data)
+	if err != nil {
+		exit(fmt.Errorf("error writing %s: %v", name, err))
+		return
+	}
+}

eth/tracers/api.go

@@ -889,11 +889,11 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *
 		return nil, err
 	}
 	defer release()
+
 	msg, err := core.TransactionToMessage(tx, types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time()), block.BaseFee())
 	if err != nil {
 		return nil, err
 	}
-
 	txctx := &Context{
 		BlockHash:   blockHash,
 		BlockNumber: block.Number(),

eth/tracers/logger/logger.go

@@ -179,8 +179,12 @@ func (s *StructLog) toLegacyJSON() json.RawMessage {
 	}
 	if len(s.Memory) > 0 {
 		memory := make([]string, 0, (len(s.Memory)+31)/32)
-		for i := 0; i+32 <= len(s.Memory); i += 32 {
+		for i := 0; i < len(s.Memory); i += 32 {
-			memory = append(memory, fmt.Sprintf("%x", s.Memory[i:i+32]))
+			end := i + 32
+			if end > len(s.Memory) {
+				end = len(s.Memory)
+			}
+			memory = append(memory, fmt.Sprintf("%x", s.Memory[i:end]))
 		}
 		msg.Memory = &memory
 	}

ethclient/gethclient/gethclient.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 )
@@ -204,6 +205,17 @@ func (ec *Client) SubscribePendingTransactions(ctx context.Context, ch chan<- co
 	return ec.c.EthSubscribe(ctx, ch, "newPendingTransactions")
 }
 
+// TraceTransaction returns the structured logs created during the execution of EVM
+// and returns them as a JSON object.
+func (ec *Client) TraceTransaction(ctx context.Context, hash common.Hash, config *tracers.TraceConfig) (any, error) {
+	var result any
+	err := ec.c.CallContext(ctx, &result, "debug_traceTransaction", hash.Hex(), config)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
 func toBlockNumArg(number *big.Int) string {
 	if number == nil {
 		return "latest"

ethclient/gethclient/gethclient_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/filters"
+	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
@@ -47,13 +48,16 @@ var (
 	testSlot     = common.HexToHash("0xdeadbeef")
 	testValue    = crypto.Keccak256Hash(testSlot[:])
 	testBalance  = big.NewInt(2e15)
+	testTxHashes []common.Hash
 )
 
 func newTestBackend(t *testing.T) (*node.Node, []*types.Block) {
 	// Generate test chain.
 	genesis, blocks := generateTestChain()
 	// Create node
-	n, err := node.New(&node.Config{})
+	n, err := node.New(&node.Config{
+		HTTPModules: []string{"debug", "eth", "admin"},
+	})
 	if err != nil {
 		t.Fatalf("can't create new node: %v", err)
 	}
@@ -63,6 +67,8 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) {
 	if err != nil {
 		t.Fatalf("can't create new ethereum service: %v", err)
 	}
+	n.RegisterAPIs(tracers.APIs(ethservice.APIBackend))
+
 	filterSystem := filters.NewFilterSystem(ethservice.APIBackend, filters.Config{})
 	n.RegisterAPIs([]rpc.API{{
 		Namespace: "eth",
@@ -93,6 +99,19 @@ func generateTestChain() (*core.Genesis, []*types.Block) {
 	generate := func(i int, g *core.BlockGen) {
 		g.OffsetTime(5)
 		g.SetExtra([]byte("test"))
+
+		to := common.BytesToAddress([]byte{byte(i + 1)})
+		tx := types.NewTx(&types.LegacyTx{
+			Nonce:    uint64(i),
+			To:       &to,
+			Value:    big.NewInt(int64(2*i + 1)),
+			Gas:      params.TxGas,
+			GasPrice: big.NewInt(params.InitialBaseFee),
+			Data:     nil,
+		})
+		tx, _ = types.SignTx(tx, types.LatestSignerForChainID(genesis.Config.ChainID), testKey)
+		g.AddTx(tx)
+		testTxHashes = append(testTxHashes, tx.Hash())
 	}
 	_, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 1, generate)
 	blocks = append([]*types.Block{genesis.ToBlock()}, blocks...)
@@ -136,9 +155,6 @@ func TestGethClient(t *testing.T) {
 		}, {
 			"TestSubscribePendingTxHashes",
 			func(t *testing.T) { testSubscribePendingTransactions(t, client) },
-		}, {
-			"TestSubscribePendingTxs",
-			func(t *testing.T) { testSubscribeFullPendingTransactions(t, client) },
 		}, {
 			"TestCallContract",
 			func(t *testing.T) { testCallContract(t, client) },
@@ -153,7 +169,12 @@ func TestGethClient(t *testing.T) {
 		{
 			"TestAccessList",
 			func(t *testing.T) { testAccessList(t, client) },
-		}, {
+		},
+		{
+			"TestTraceTransaction",
+			func(t *testing.T) { testTraceTransactions(t, client) },
+		},
+		{
 			"TestSetHead",
 			func(t *testing.T) { testSetHead(t, client) },
 		},
@@ -197,7 +218,7 @@ func testAccessList(t *testing.T, client *rpc.Client) {
 			wantVMErr: "execution reverted",
 			wantAL: `[
   {
-    "address": "0x3a220f351252089d385b29beca14e27f204c296a",
+    "address": "0xdb7d6ab1f17c6b31909ae466702703daef9269cf",
     "storageKeys": [
       "0x0000000000000000000000000000000000000000000000000000000000000081"
     ]
@@ -389,16 +410,26 @@ func testSetHead(t *testing.T, client *rpc.Client) {
 func testSubscribePendingTransactions(t *testing.T, client *rpc.Client) {
 	ec := New(client)
 	ethcl := ethclient.NewClient(client)
+
 	// Subscribe to Transactions
-	ch := make(chan common.Hash)
+	ch1 := make(chan common.Hash)
-	ec.SubscribePendingTransactions(context.Background(), ch)
+	ec.SubscribePendingTransactions(context.Background(), ch1)
+
+	// Subscribe to Transactions
+	ch2 := make(chan *types.Transaction)
+	ec.SubscribeFullPendingTransactions(context.Background(), ch2)
+
 	// Send a transaction
 	chainID, err := ethcl.ChainID(context.Background())
 	if err != nil {
 		t.Fatal(err)
 	}
+	nonce, err := ethcl.NonceAt(context.Background(), testAddr, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Create transaction
-	tx := types.NewTransaction(0, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil)
+	tx := types.NewTransaction(nonce, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil)
 	signer := types.LatestSignerForChainID(chainID)
 	signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey)
 	if err != nil {
@@ -414,41 +445,12 @@ func testSubscribePendingTransactions(t *testing.T, client *rpc.Client) {
 		t.Fatal(err)
 	}
 	// Check that the transaction was sent over the channel
-	hash := <-ch
+	hash := <-ch1
 	if hash != signedTx.Hash() {
 		t.Fatalf("Invalid tx hash received, got %v, want %v", hash, signedTx.Hash())
 	}
-}
-
-func testSubscribeFullPendingTransactions(t *testing.T, client *rpc.Client) {
-	ec := New(client)
-	ethcl := ethclient.NewClient(client)
-	// Subscribe to Transactions
-	ch := make(chan *types.Transaction)
-	ec.SubscribeFullPendingTransactions(context.Background(), ch)
-	// Send a transaction
-	chainID, err := ethcl.ChainID(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Create transaction
-	tx := types.NewTransaction(1, common.Address{1}, big.NewInt(1), 22000, big.NewInt(1), nil)
-	signer := types.LatestSignerForChainID(chainID)
-	signature, err := crypto.Sign(signer.Hash(tx).Bytes(), testKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-	signedTx, err := tx.WithSignature(signer, signature)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Send transaction
-	err = ethcl.SendTransaction(context.Background(), signedTx)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Check that the transaction was sent over the channel
-	tx = <-ch
+	tx = <-ch2
 	if tx.Hash() != signedTx.Hash() {
 		t.Fatalf("Invalid tx hash received, got %v, want %v", tx.Hash(), signedTx.Hash())
 	}
@@ -478,6 +480,25 @@ func testCallContract(t *testing.T, client *rpc.Client) {
 	}
 }
 
+func testTraceTransactions(t *testing.T, client *rpc.Client) {
+	ec := New(client)
+	for _, txHash := range testTxHashes {
+		// Struct logger
+		_, err := ec.TraceTransaction(context.Background(), txHash, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Struct logger
+		_, err = ec.TraceTransaction(context.Background(), txHash,
+			&tracers.TraceConfig{},
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
 func TestOverrideAccountMarshal(t *testing.T) {
 	om := map[common.Address]OverrideAccount{
 		{0x11}: {