diff --git a/cmd/era/main.go b/cmd/era/main.go index 35a889d4dc..1c26f44ad4 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -30,6 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/params" @@ -53,7 +55,7 @@ var ( eraSizeFlag = &cli.IntFlag{ Name: "size", Usage: "number of blocks per era", - Value: era.MaxEra1Size, + Value: era.MaxSize, } txsFlag = &cli.BoolFlag{ Name: "txs", @@ -131,7 +133,7 @@ func block(ctx *cli.Context) error { return nil } -// info prints some high-level information about the era1 file. +// info prints some high-level information about the era file. func info(ctx *cli.Context) error { epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) if err != nil { @@ -142,33 +144,34 @@ func info(ctx *cli.Context) error { return err } defer e.Close() - acc, err := e.Accumulator() - if err != nil { - return fmt.Errorf("error reading accumulator: %w", err) + var ( + accHex string + tdStr string + ) + if acc, err := e.Accumulator(); err == nil { + accHex = acc.Hex() } - td, err := e.InitialTD() - if err != nil { - return fmt.Errorf("error reading total difficulty: %w", err) + if td, err := e.InitialTD(); err == nil { + tdStr = td.String() } info := struct { - Accumulator common.Hash `json:"accumulator"` - TotalDifficulty *big.Int `json:"totalDifficulty"` - StartBlock uint64 `json:"startBlock"` - Count uint64 `json:"count"` + Accumulator string `json:"accumulator,omitempty"` + TotalDifficulty string `json:"totalDifficulty,omitempty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` }{ - acc, td, e.Start(), e.Count(), + accHex, tdStr, e.Start(), e.Count(), } b, _ := json.MarshalIndent(info, "", " ") fmt.Println(string(b)) return nil } -// open opens an era1 file at a certain epoch. -func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { - var ( - dir = ctx.String(dirFlag.Name) - network = ctx.String(networkFlag.Name) - ) +// open opens an era file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (era.Era, error) { + dir := ctx.String(dirFlag.Name) + network := ctx.String(networkFlag.Name) + entries, err := era.ReadDir(dir, network) if err != nil { return nil, fmt.Errorf("error reading era dir: %w", err) @@ -176,7 +179,28 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return era.Open(filepath.Join(dir, entries[epoch])) + path := filepath.Join(dir, entries[epoch]) + return openByPath(path) +} + +// openByPath tries to open a single file as either eraE or era1 based on extension, +// falling back to the other reader if needed. +func openByPath(path string) (era.Era, error) { + switch strings.ToLower(filepath.Ext(path)) { + case ".erae": + if e, err := execdb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + case ".era1": + if e, err := onedb.Open(path); err != nil { + return nil, err + } else { + return e, nil + } + } + return nil, fmt.Errorf("unsupported or unreadable era file: %s", path) } // verify checks each era1 file in a directory to ensure it is well-formed and @@ -203,18 +227,58 @@ func verify(ctx *cli.Context) error { return fmt.Errorf("error reading %s: %w", dir, err) } - if len(entries) != len(roots) { - return errors.New("number of era1 files should match the number of accumulator hashes") + // Build the verification list respecting the rule: + // era1: must have accumulator, always verify + // erae: verify only if accumulator exists (pre-merge) + + // Build list of files to verify. + verify := make([]string, 0, len(entries)) + + for _, name := range entries { + path := filepath.Join(dir, name) + ext := strings.ToLower(filepath.Ext(name)) + + switch ext { + case ".era1": + e, err := onedb.Open(path) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr != nil { + return fmt.Errorf("era1 file %s missing accumulator: %w", name, accErr) + } + verify = append(verify, path) + + case ".erae": + e, err := execdb.Open(path) + if err != nil { + return fmt.Errorf("error opening erae file %s: %w", name, err) + } + _, accErr := e.Accumulator() + e.Close() + if accErr == nil { + verify = append(verify, path) // pre-merge only + } + default: + return fmt.Errorf("unsupported era file: %s", name) + } + } + + if len(verify) != len(roots) { + return fmt.Errorf("mismatch between eras to verify (%d) and provided roots (%d)", len(verify), len(roots)) } // Verify each epoch matches the expected root. for i, want := range roots { // Wrap in function so defers don't stack. err := func() error { - name := entries[i] - e, err := era.Open(filepath.Join(dir, name)) + path := verify[i] + name := filepath.Base(path) + e, err := openByPath(path) if err != nil { - return fmt.Errorf("error opening era1 file %s: %w", name, err) + return fmt.Errorf("error opening era file %s: %w", name, err) } defer e.Close() // Read accumulator and check against expected. @@ -243,7 +307,7 @@ func verify(ctx *cli.Context) error { } // checkAccumulator verifies the accumulator matches the data in the Era. -func checkAccumulator(e *era.Era) error { +func checkAccumulator(e era.Era) error { var ( err error want common.Hash @@ -257,7 +321,7 @@ func checkAccumulator(e *era.Era) error { if td, err = e.InitialTD(); err != nil { return fmt.Errorf("error reading total difficulty: %w", err) } - it, err := era.NewIterator(e) + it, err := e.Iterator() if err != nil { return fmt.Errorf("error making era iterator: %w", err) } @@ -290,9 +354,13 @@ func checkAccumulator(e *era.Era) error { if rr != block.ReceiptHash() { return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) } - hashes = append(hashes, block.Hash()) - td.Add(td, block.Difficulty()) - tds = append(tds, new(big.Int).Set(td)) + // Only include pre-merge blocks in accumulator calculation. + // Post-merge blocks have difficulty == 0. + if block.Difficulty().Sign() > 0 { + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } } if it.Error() != nil { return fmt.Errorf("error reading block %d: %w", it.Number(), it.Error()) diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 0f39df0753..4ba7b5f130 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -115,9 +115,6 @@ func Transaction(ctx *cli.Context) error { } var results []result for it.Next() { - if err := it.Err(); err != nil { - return NewError(ErrorIO, err) - } var tx types.Transaction err := rlp.DecodeBytes(it.Value(), &tx) if err != nil { @@ -188,6 +185,10 @@ func Transaction(ctx *cli.Context) error { } results = append(results, r) } + if err := it.Err(); err != nil { + return NewError(ErrorIO, err) + } + out, err := json.MarshalIndent(results, "", " ") fmt.Println(string(out)) return err diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9d04dd0f1b..1ccb78d622 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "regexp" @@ -43,6 +44,8 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/eradl" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -153,7 +156,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -164,7 +167,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: utils.DatabaseFlags, + Flags: slices.Concat([]cli.Flag{utils.EraFormatFlag}, utils.DatabaseFlags), Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -516,15 +519,27 @@ func importHistory(ctx *cli.Context) error { network = networks[0] } - if err := utils.ImportHistory(chain, dir, network); err != nil { + var ( + format = ctx.String(utils.EraFormatFlag.Name) + from func(era.ReadAtSeekCloser) (era.Era, error) + ) + switch format { + case "era1", "era": + from = onedb.From + case "erae": + from = execdb.From + default: + return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) + } + if err := utils.ImportHistory(chain, dir, network, from); err != nil { return err } + fmt.Printf("Import done in %v\n", time.Since(start)) return nil } -// exportHistory exports chain history in Era archives at a specified -// directory. +// exportHistory exports chain history in Era archives at a specified directory. func exportHistory(ctx *cli.Context) error { if ctx.Args().Len() != 3 { utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) @@ -550,10 +565,26 @@ func exportHistory(ctx *cli.Context) error { if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) } - err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) - if err != nil { + + var ( + format = ctx.String(utils.EraFormatFlag.Name) + filename func(network string, epoch int, root common.Hash) string + newBuilder func(w io.Writer) era.Builder + ) + switch format { + case "era1", "era": + newBuilder = func(w io.Writer) era.Builder { return onedb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return onedb.Filename(network, epoch, root) } + case "erae": + newBuilder = func(w io.Writer) era.Builder { return execdb.NewBuilder(w) } + filename = func(network string, epoch int, root common.Hash) string { return execdb.Filename(network, epoch, root) } + default: + return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format) + } + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), newBuilder, filename); err != nil { utils.Fatalf("Export error: %v\n", err) } + fmt.Printf("Export done in %v\n", time.Since(start)) return nil } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 87627467d2..7943d95d65 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/telemetry/tracesetup" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -239,9 +240,15 @@ func makeFullNode(ctx *cli.Context) *node.Node { cfg.Eth.OverrideVerkle = &v } - // Start metrics export if enabled + // Start metrics export if enabled. utils.SetupMetrics(&cfg.Metrics) + // Setup OpenTelemetry reporting if enabled. + if err := tracesetup.SetupTelemetry(cfg.Node.OpenTelemetry, stack); err != nil { + utils.Fatalf("failed to setup OpenTelemetry: %v", err) + } + + // Add Ethereum service. backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Create gauge with geth system and build information @@ -398,9 +405,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + utils.Fatalf("Flags --%s, --%s, --%s are only available for influxdb-v2", utils.MetricsInfluxDBOrganizationFlag.Name, utils.MetricsInfluxDBTokenFlag.Name, utils.MetricsInfluxDBBucketFlag.Name) } else if enableExportV2 && v1FlagIsSet { - utils.Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + utils.Fatalf("Flags --%s, --%s are only available for influxdb-v1", utils.MetricsInfluxDBUsernameFlag.Name, utils.MetricsInfluxDBPasswordFlag.Name) } } } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9aabaaba98..2291e0aafa 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -196,6 +196,13 @@ var ( utils.RPCTxSyncDefaultTimeoutFlag, utils.RPCTxSyncMaxTimeoutFlag, utils.RPCGlobalRangeLimitFlag, + utils.RPCTelemetryFlag, + utils.RPCTelemetryEndpointFlag, + utils.RPCTelemetryUserFlag, + utils.RPCTelemetryPasswordFlag, + utils.RPCTelemetryInstanceIDFlag, + utils.RPCTelemetryTagsFlag, + utils.RPCTelemetrySampleRatioFlag, } metricsFlags = []cli.Flag{ diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index ff7acbad36..388b2e0610 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -38,8 +38,8 @@ require ( go.opentelemetry.io/otel v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/sync v0.12.0 // indirect + golang.org/x/crypto v0.44.0 // indirect + golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.39.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index c07f9e9a65..9dfb0a65f2 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -123,22 +123,22 @@ go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3e337a3d00..995724e6fc 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -57,6 +57,8 @@ const ( importBatchSize = 2500 ) +type EraFileFormat int + // ErrImportInterrupted is returned when the user interrupts the import process. var ErrImportInterrupted = errors.New("interrupted") @@ -250,7 +252,7 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string) error { +func ImportHistory(chain *core.BlockChain, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { return errors.New("history import only supported when starting from genesis") } @@ -263,42 +265,49 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { return fmt.Errorf("unable to read checksums.txt: %w", err) } if len(checksums) != len(entries) { - return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", + len(checksums), len(entries)) } + var ( start = time.Now() reported = time.Now() imported = 0 h = sha256.New() - buf = bytes.NewBuffer(nil) + scratch = bytes.NewBuffer(nil) ) - for i, filename := range entries { + + for i, file := range entries { err := func() error { - f, err := os.Open(filepath.Join(dir, filename)) + path := filepath.Join(dir, file) + + // validate against checksum file in directory + f, err := os.Open(path) if err != nil { - return fmt.Errorf("unable to open era: %w", err) + return fmt.Errorf("open %s: %w", path, err) } defer f.Close() - - // Validate checksum. if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to recalculate checksum: %w", err) - } - if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { - return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + return fmt.Errorf("checksum %s: %w", path, err) } + got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex() + want := checksums[i] h.Reset() - buf.Reset() + scratch.Reset() + if got != want { + return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) + } // Import all block data from Era1. - e, err := era.From(f) + e, err := from(f) if err != nil { return fmt.Errorf("error opening era: %w", err) } - it, err := era.NewIterator(e) + it, err := e.Iterator() if err != nil { - return fmt.Errorf("error making era reader: %w", err) + return fmt.Errorf("error creating iterator: %w", err) } + for it.Next() { block, err := it.Block() if err != nil { @@ -311,26 +320,28 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) } - encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) - if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil { + enc := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) + if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } - imported += 1 + imported++ - // Give the user some feedback that something is happening. if time.Since(reported) >= 8*time.Second { - log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Importing Era files", "head", it.Number(), "imported", imported, + "elapsed", common.PrettyDuration(time.Since(start))) imported = 0 reported = time.Now() } } + if err := it.Error(); err != nil { + return err + } return nil }() if err != nil { return err } } - return nil } @@ -389,7 +400,6 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return err } defer fh.Close() - var writer io.Writer = fh if strings.HasSuffix(fn, ".gz") { writer = gzip.NewWriter(writer) @@ -405,7 +415,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las // ExportHistory exports blockchain history into the specified directory, // following the Era format. -func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { +func ExportHistory(bc *core.BlockChain, dir string, first, last uint64, newBuilder func(io.Writer) era.Builder, filename func(network string, epoch int, lastBlockHash common.Hash) string) error { log.Info("Exporting blockchain history", "dir", dir) if head := bc.CurrentBlock().Number.Uint64(); head < last { log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) @@ -418,76 +428,100 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er if err := os.MkdirAll(dir, os.ModePerm); err != nil { return fmt.Errorf("error creating output directory: %w", err) } + var ( start = time.Now() reported = time.Now() h = sha256.New() buf = bytes.NewBuffer(nil) + td = new(big.Int) checksums []string ) - td := new(big.Int) - for i := uint64(0); i < first; i++ { - td.Add(td, bc.GetHeaderByNumber(i).Difficulty) + + // Compute initial TD by accumulating difficulty from genesis to first-1. + // This is necessary because TD is no longer stored in the database. Only + // compute if a segment of the export is pre-merge. + b := bc.GetBlockByNumber(first) + if b == nil { + return fmt.Errorf("block #%d not found", first) } - for i := first; i <= last; i += step { - err := func() error { - filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{})) - f, err := os.Create(filename) + if first > 0 && b.Difficulty().Sign() != 0 { + log.Info("Computing initial total difficulty", "from", 0, "to", first-1) + for i := uint64(0); i < first; i++ { + b := bc.GetBlockByNumber(i) + if b == nil { + return fmt.Errorf("block #%d not found while computing initial TD", i) + } + td.Add(td, b.Difficulty()) + } + log.Info("Initial total difficulty computed", "td", td) + } + + for batch := first; batch <= last; batch += uint64(era.MaxSize) { + idx := int(batch / uint64(era.MaxSize)) + tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{})) + + if err := func() error { + f, err := os.Create(tmpPath) if err != nil { - return fmt.Errorf("could not create era file: %w", err) + return err } defer f.Close() - w := era.NewBuilder(f) - for j := uint64(0); j < step && j <= last-i; j++ { - var ( - n = i + j - block = bc.GetBlockByNumber(n) - ) + builder := newBuilder(f) + + for j := uint64(0); j < uint64(era.MaxSize) && batch+j <= last; j++ { + n := batch + j + block := bc.GetBlockByNumber(n) if block == nil { - return fmt.Errorf("export failed on #%d: not found", n) + return fmt.Errorf("block #%d not found", n) } - receipts := bc.GetReceiptsByHash(block.Hash()) - if receipts == nil { - return fmt.Errorf("export failed on #%d: receipts not found", n) + receipt := bc.GetReceiptsByHash(block.Hash()) + if receipt == nil { + return fmt.Errorf("receipts for #%d missing", n) } - td.Add(td, block.Difficulty()) - if err := w.Add(block, receipts, new(big.Int).Set(td)); err != nil { + + // For pre-merge blocks, pass accumulated TD. + // For post-merge blocks (difficulty == 0), pass nil TD. + var blockTD *big.Int + if block.Difficulty().Sign() != 0 { + td.Add(td, block.Difficulty()) + blockTD = new(big.Int).Set(td) + } + + if err := builder.Add(block, receipt, blockTD); err != nil { return err } } - root, err := w.Finalize() + id, err := builder.Finalize() if err != nil { - return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + return err } - // Set correct filename with root. - os.Rename(filename, filepath.Join(dir, era.Filename(network, int(i/step), root))) - - // Compute checksum of entire Era1. if _, err := f.Seek(0, io.SeekStart); err != nil { return err } - if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to calculate checksum: %w", err) - } - checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) h.Reset() buf.Reset() - return nil - }() - if err != nil { + if _, err := io.Copy(h, f); err != nil { + return err + } + checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) + + // Close before rename. It's required on Windows. + f.Close() + final := filepath.Join(dir, filename(network, idx, id)) + return os.Rename(tmpPath, final) + }(); err != nil { return err } + if time.Since(reported) >= 8*time.Second { - log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("export progress", "exported", batch, "elapsed", common.PrettyDuration(time.Since(start))) reported = time.Now() } } - os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) - - log.Info("Exported blockchain to", "dir", dir) - + _ = os.WriteFile(filepath.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) return nil } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 91448c520c..3cb365b108 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1042,6 +1042,55 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Value: metrics.DefaultConfig.InfluxDBOrganization, Category: flags.MetricsCategory, } + + // RPC Telemetry + RPCTelemetryFlag = &cli.BoolFlag{ + Name: "rpc.telemetry", + Usage: "Enable RPC telemetry", + Category: flags.APICategory, + } + + RPCTelemetryEndpointFlag = &cli.StringFlag{ + Name: "rpc.telemetry.endpoint", + Usage: "Defines where RPC telemetry is sent (e.g., http://localhost:4318)", + Category: flags.APICategory, + } + + RPCTelemetryUserFlag = &cli.StringFlag{ + Name: "rpc.telemetry.username", + Usage: "HTTP Basic Auth username for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryPasswordFlag = &cli.StringFlag{ + Name: "rpc.telemetry.password", + Usage: "HTTP Basic Auth password for OpenTelemetry", + Category: flags.APICategory, + } + + RPCTelemetryInstanceIDFlag = &cli.StringFlag{ + Name: "rpc.telemetry.instance-id", + Usage: "OpenTelemetry instance ID", + Category: flags.APICategory, + } + + RPCTelemetryTagsFlag = &cli.StringFlag{ + Name: "rpc.telemetry.tags", + Usage: "Comma-separated tags (key/values) added as attributes to the OpenTelemetry resource struct", + Category: flags.APICategory, + } + + RPCTelemetrySampleRatioFlag = &cli.Float64Flag{ + Name: "rpc.telemetry.sample-ratio", + Usage: "Defines the sampling ratio for RPC telemetry (0.0 to 1.0)", + Value: 1.0, + Category: flags.APICategory, + } + // Era flags are a group of flags related to the era archive format. + EraFormatFlag = &cli.StringFlag{ + Name: "era.format", + Usage: "Archive format: 'era1' or 'erae'", + } ) var ( @@ -1432,6 +1481,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setNodeUserIdent(ctx, cfg) SetDataDir(ctx, cfg) setSmartCard(ctx, cfg) + setOpenTelemetry(ctx, cfg) if ctx.IsSet(JWTSecretFlag.Name) { cfg.JWTSecret = ctx.String(JWTSecretFlag.Name) @@ -1499,6 +1549,33 @@ func setSmartCard(ctx *cli.Context, cfg *node.Config) { cfg.SmartCardDaemonPath = path } +func setOpenTelemetry(ctx *cli.Context, cfg *node.Config) { + tcfg := &cfg.OpenTelemetry + if ctx.IsSet(RPCTelemetryFlag.Name) { + tcfg.Enabled = ctx.Bool(RPCTelemetryFlag.Name) + } + if ctx.IsSet(RPCTelemetryEndpointFlag.Name) { + tcfg.Endpoint = ctx.String(RPCTelemetryEndpointFlag.Name) + } + if ctx.IsSet(RPCTelemetryUserFlag.Name) { + tcfg.AuthUser = ctx.String(RPCTelemetryUserFlag.Name) + } + if ctx.IsSet(RPCTelemetryPasswordFlag.Name) { + tcfg.AuthPassword = ctx.String(RPCTelemetryPasswordFlag.Name) + } + if ctx.IsSet(RPCTelemetryInstanceIDFlag.Name) { + tcfg.InstanceID = ctx.String(RPCTelemetryInstanceIDFlag.Name) + } + if ctx.IsSet(RPCTelemetryTagsFlag.Name) { + tcfg.Tags = ctx.String(RPCTelemetryTagsFlag.Name) + } + tcfg.SampleRatio = ctx.Float64(RPCTelemetrySampleRatioFlag.Name) + + if tcfg.Endpoint != "" && !tcfg.Enabled { + log.Warn(fmt.Sprintf("OpenTelemetry endpoint configured but telemetry is not enabled, use --%s to enable.", RPCTelemetryFlag.Name)) + } +} + func SetDataDir(ctx *cli.Context, cfg *node.Config) { switch { case ctx.IsSet(DataDirFlag.Name): diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 994756eda5..6631946129 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -33,6 +33,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -44,136 +46,148 @@ var ( ) func TestHistoryImportAndExport(t *testing.T) { - var ( - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - genesis = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, - } - signer = types.LatestSigner(genesis.Config) - ) - - // Generate chain. - db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { - if i == 0 { - return - } - tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ - ChainID: genesis.Config.ChainID, - Nonce: uint64(i - 1), - GasTipCap: common.Big0, - GasFeeCap: g.PrevBlock(0).BaseFee(), - Gas: 50000, - To: &common.Address{0xaa}, - Value: big.NewInt(int64(i)), - Data: nil, - AccessList: nil, - }) - if err != nil { - t.Fatalf("error creating tx: %v", err) - } - g.AddTx(tx) - }) - - // Initialize BlockChain. - chain, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if _, err := chain.InsertChain(blocks); err != nil { - t.Fatalf("error inserting chain: %v", err) - } - - // Make temp directory for era files. - dir := t.TempDir() - - // Export history to temp directory. - if err := ExportHistory(chain, dir, 0, count, step); err != nil { - t.Fatalf("error exporting history: %v", err) - } - - // Read checksums. - b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) - if err != nil { - t.Fatalf("failed to read checksums: %v", err) - } - checksums := strings.Split(string(b), "\n") - - // Verify each Era. - entries, _ := era.ReadDir(dir, "mainnet") - for i, filename := range entries { - func() { - f, err := os.Open(filepath.Join(dir, filename)) - if err != nil { - t.Fatalf("error opening era file: %v", err) - } + for _, tt := range []struct { + name string + builder func(io.Writer) era.Builder + filename func(network string, epoch int, root common.Hash) string + from func(f era.ReadAtSeekCloser) (era.Era, error) + }{ + {"era1", onedb.NewBuilder, onedb.Filename, onedb.From}, + {"erae", execdb.NewBuilder, execdb.Filename, execdb.From}, + } { + t.Run(tt.name, func(t *testing.T) { var ( - h = sha256.New() - buf = bytes.NewBuffer(nil) + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) ) - if _, err := io.Copy(h, f); err != nil { - t.Fatalf("unable to recalculate checksum: %v", err) - } - if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { - t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) - } - e, err := era.From(f) - if err != nil { - t.Fatalf("error opening era: %v", err) - } - defer e.Close() - it, err := era.NewIterator(e) - if err != nil { - t.Fatalf("error making era reader: %v", err) - } - for j := 0; it.Next(); j++ { - n := i*int(step) + j - if it.Error() != nil { - t.Fatalf("error reading block entry %d: %v", n, it.Error()) + + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return } - block, receipts, err := it.BlockAndReceipts() + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) if err != nil { - t.Fatalf("error reading block entry %d: %v", n, err) - } - want := chain.GetBlockByNumber(uint64(n)) - if want, got := uint64(n), block.NumberU64(); want != got { - t.Fatalf("blocks out of order: want %d, got %d", want, got) - } - if want.Hash() != block.Hash() { - t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) - } - if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { - t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) - } - if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { - t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) - } - if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { - t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + t.Fatalf("error creating tx: %v", err) } + g.AddTx(tx) + }) + + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error inserting chain: %v", err) } - }() - } - // Now import Era. - db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) - if err != nil { - panic(err) - } - t.Cleanup(func() { - db2.Close() - }) + // Make temp directory for era files. + dir := t.TempDir() - genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) - imported, err := core.NewBlockChain(db2, genesis, ethash.NewFaker(), nil) - if err != nil { - t.Fatalf("unable to initialize chain: %v", err) - } - if err := ImportHistory(imported, dir, "mainnet"); err != nil { - t.Fatalf("failed to import chain: %v", err) - } - if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { - t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, tt.builder, tt.filename); err != nil { + t.Fatalf("error exporting history: %v", err) + } + + // Read checksums. + b, err := os.ReadFile(filepath.Join(dir, "checksums.txt")) + if err != nil { + t.Fatalf("failed to read checksums: %v", err) + } + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + func() { + f, err := os.Open(filepath.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + e, err := tt.from(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + defer e.Close() + it, err := e.Iterator() + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, it.Error()) + } + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) + } + } + }() + } + + // Now import Era. + db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{}) + if err != nil { + panic(err) + } + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults)) + imported, err := core.NewBlockChain(db2, genesis, ethash.NewFaker(), nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil { + t.Fatalf("failed to import chain: %v", err) + } + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } + }) } } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 6ae64fb2fd..14308dd698 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -424,7 +424,13 @@ func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but bodies can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false @@ -466,7 +472,13 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // to a block. func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - return true + // Block is in ancient store, but receipts can be pruned. + // Check if the block number is above the pruning tail. + tail, _ := db.Tail() + if number >= tail { + return true + } + return false } if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { return false diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 10eb454015..8c8c3ec9bb 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -147,9 +147,6 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex := uint64(0) for iter.Next() { - if iter.Err() != nil { - return nil, 0, iter.Err() - } // The preimage for the hash calculation of legacy transactions // is just their RLP encoding. For typed (EIP-2718) transactions, // which are encoded as byte arrays, the preimage is the content of @@ -171,6 +168,9 @@ func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Trans } txIndex++ } + if iter.Err() != nil { + return nil, 0, iter.Err() + } return nil, 0, errors.New("transaction not found") } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 0ed974b745..8c6b18df08 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -166,6 +166,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s if err != nil { return err } + defer table.Close() table.dumpIndexStdout(start, end) return nil } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index f846d4d16c..afa1aa7a4c 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -357,9 +357,9 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch } select { case <-interrupt: - logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", nextNum, "elapsed", common.PrettyDuration(time.Since(start))) } } diff --git a/core/rawdb/eradb/eradb.go b/core/rawdb/eradb/eradb.go index a552b94da9..d715c824ed 100644 --- a/core/rawdb/eradb/eradb.go +++ b/core/rawdb/eradb/eradb.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) @@ -51,7 +52,7 @@ type Store struct { type fileCacheEntry struct { refcount int // reference count. This is protected by Store.mu! opened chan struct{} // signals opening of file has completed - file *era.Era // the file + file *onedb.Era // the file err error // error from opening the file } @@ -102,7 +103,7 @@ func (db *Store) Close() { // GetRawBody returns the raw body for a given block number. func (db *Store) GetRawBody(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -117,7 +118,7 @@ func (db *Store) GetRawBody(number uint64) ([]byte, error) { // GetRawReceipts returns the raw receipts for a given block number. func (db *Store) GetRawReceipts(number uint64) ([]byte, error) { - epoch := number / uint64(era.MaxEra1Size) + epoch := number / uint64(era.MaxSize) entry := db.getEraByEpoch(epoch) if entry.err != nil { if errors.Is(entry.err, fs.ErrNotExist) { @@ -249,7 +250,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC } // fileOpened is called after an era file has been successfully opened. -func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *era.Era) { +func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) { db.mu.Lock() defer db.mu.Unlock() @@ -282,7 +283,7 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error entry.err = err } -func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { +func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) { // File name scheme is --. glob := fmt.Sprintf("*-%05d-*.era1", epoch) matches, err := filepath.Glob(filepath.Join(db.datadir, glob)) @@ -297,17 +298,17 @@ func (db *Store) openEraFile(epoch uint64) (*era.Era, error) { } filename := matches[0] - e, err := era.Open(filename) + e, err := onedb.Open(filename) if err != nil { return nil, err } // Sanity-check start block. - if e.Start()%uint64(era.MaxEra1Size) != 0 { + if e.Start()%uint64(era.MaxSize) != 0 { e.Close() - return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxEra1Size) + return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize) } log.Debug("Opened era1 file", "epoch", epoch) - return e, nil + return e.(*onedb.Era), nil } // doneWithFile signals that the caller has finished using a file. diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index f531e668c3..5494a648c8 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -221,13 +221,12 @@ func cleanup(path string) error { if err != nil { return err } + defer dir.Close() + names, err := dir.Readdirnames(0) if err != nil { return err } - if cerr := dir.Close(); cerr != nil { - return cerr - } for _, name := range names { if name == filepath.Base(path)+tmpSuffix { log.Info("Removed leftover freezer directory", "name", name) diff --git a/core/tracing/gen_balance_change_reason_stringer.go b/core/tracing/gen_balance_change_reason_stringer.go index 250b521193..f0edfad872 100644 --- a/core/tracing/gen_balance_change_reason_stringer.go +++ b/core/tracing/gen_balance_change_reason_stringer.go @@ -31,8 +31,9 @@ const _BalanceChangeReason_name = "UnspecifiedBalanceIncreaseRewardMineUncleBala var _BalanceChangeReason_index = [...]uint16{0, 11, 41, 71, 96, 125, 160, 181, 205, 231, 256, 264, 276, 303, 330, 361, 367} func (i BalanceChangeReason) String() string { - if i >= BalanceChangeReason(len(_BalanceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_BalanceChangeReason_index)-1 { return "BalanceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _BalanceChangeReason_name[_BalanceChangeReason_index[i]:_BalanceChangeReason_index[i+1]] + return _BalanceChangeReason_name[_BalanceChangeReason_index[idx]:_BalanceChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_code_change_reason_stringer.go b/core/tracing/gen_code_change_reason_stringer.go index 9372954063..2531b10471 100644 --- a/core/tracing/gen_code_change_reason_stringer.go +++ b/core/tracing/gen_code_change_reason_stringer.go @@ -22,8 +22,9 @@ const _CodeChangeReason_name = "UnspecifiedContractCreationGenesisAuthorizationA var _CodeChangeReason_index = [...]uint8{0, 11, 27, 34, 47, 65, 77, 83} func (i CodeChangeReason) String() string { - if i >= CodeChangeReason(len(_CodeChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_CodeChangeReason_index)-1 { return "CodeChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _CodeChangeReason_name[_CodeChangeReason_index[i]:_CodeChangeReason_index[i+1]] + return _CodeChangeReason_name[_CodeChangeReason_index[idx]:_CodeChangeReason_index[idx+1]] } diff --git a/core/tracing/gen_nonce_change_reason_stringer.go b/core/tracing/gen_nonce_change_reason_stringer.go index cd19200db8..8c9099b7ce 100644 --- a/core/tracing/gen_nonce_change_reason_stringer.go +++ b/core/tracing/gen_nonce_change_reason_stringer.go @@ -23,8 +23,9 @@ const _NonceChangeReason_name = "UnspecifiedGenesisEoACallContractCreatorNewCont var _NonceChangeReason_index = [...]uint8{0, 11, 18, 25, 40, 51, 64, 70, 82} func (i NonceChangeReason) String() string { - if i >= NonceChangeReason(len(_NonceChangeReason_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_NonceChangeReason_index)-1 { return "NonceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" } - return _NonceChangeReason_name[_NonceChangeReason_index[i]:_NonceChangeReason_index[i+1]] + return _NonceChangeReason_name[_NonceChangeReason_index[idx]:_NonceChangeReason_index[idx+1]] } diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 27441ac2e2..3947ba50a1 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -2096,6 +2096,11 @@ func (p *BlobPool) Clear() { p.index = make(map[common.Address][]*blobTxMeta) p.spent = make(map[common.Address]*uint256.Int) + // Reset counters and the gapped buffer + p.stored = 0 + p.gapped = make(map[common.Address][]*types.Transaction) + p.gappedSource = make(map[common.Hash]common.Address) + var ( basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head.Load())) blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) diff --git a/core/types/receipt.go b/core/types/receipt.go index 5b6669f274..ba7d9900f0 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -424,3 +424,33 @@ func EncodeBlockReceiptLists(receipts []Receipts) []rlp.RawValue { } return result } + +// SlimReceipt is a wrapper around a Receipt with RLP serialization that omits +// the Bloom field and includes the tx type. Used for era files. +type SlimReceipt Receipt + +type slimReceiptRLP struct { + Type uint8 + StatusEncoding []byte + CumulativeGasUsed uint64 + Logs []*Log +} + +// EncodeRLP implements rlp.Encoder, encoding the receipt as +// [tx-type, post-state-or-status, cumulative-gas, logs]. +func (r *SlimReceipt) EncodeRLP(w io.Writer) error { + data := &slimReceiptRLP{r.Type, (*Receipt)(r).statusEncoding(), r.CumulativeGasUsed, r.Logs} + return rlp.Encode(w, data) +} + +// DecodeRLP implements rlp.Decoder. +func (r *SlimReceipt) DecodeRLP(s *rlp.Stream) error { + var data slimReceiptRLP + if err := s.Decode(&data); err != nil { + return err + } + r.Type = data.Type + r.CumulativeGasUsed = data.CumulativeGasUsed + r.Logs = data.Logs + return (*Receipt)(r).setStatus(data.StatusEncoding) +} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 8f805ff096..676d9c3d30 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -512,6 +512,45 @@ func TestReceiptUnmarshalBinary(t *testing.T) { } } +func TestSlimReceiptEncodingDecoding(t *testing.T) { + tests := []*Receipt{ + legacyReceipt, + accessListReceipt, + eip1559Receipt, + { + Type: BlobTxType, + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 100, + Logs: []*Log{}, + }, + } + for i, want := range tests { + enc, err := rlp.EncodeToBytes((*SlimReceipt)(want)) + if err != nil { + t.Fatalf("test %d: encode error: %v", i, err) + } + got := new(SlimReceipt) + if err := rlp.DecodeBytes(enc, got); err != nil { + t.Fatalf("test %d: decode error: %v", i, err) + } + if got.Type != want.Type { + t.Errorf("test %d: Type mismatch: got %d, want %d", i, got.Type, want.Type) + } + if got.Status != want.Status { + t.Errorf("test %d: Status mismatch: got %d, want %d", i, got.Status, want.Status) + } + if !bytes.Equal(got.PostState, want.PostState) { + t.Errorf("test %d: PostState mismatch: got %x, want %x", i, got.PostState, want.PostState) + } + if got.CumulativeGasUsed != want.CumulativeGasUsed { + t.Errorf("test %d: CumulativeGasUsed mismatch: got %d, want %d", i, got.CumulativeGasUsed, want.CumulativeGasUsed) + } + if len(got.Logs) != len(want.Logs) { + t.Errorf("test %d: Logs length mismatch: got %d, want %d", i, len(got.Logs), len(want.Logs)) + } + } +} + func clearComputedFieldsOnReceipts(receipts []*Receipt) []*Receipt { r := make([]*Receipt, len(receipts)) for i, receipt := range receipts { diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 56cb2686a6..3f776146f1 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -19,6 +19,7 @@ package vm import ( "bytes" "encoding/json" + "errors" "fmt" "math/big" "os" @@ -1013,10 +1014,11 @@ func TestEIP8024_Execution(t *testing.T) { evm := NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) tests := []struct { - name string - codeHex string - wantErr bool - wantVals []uint64 + name string + codeHex string + wantErr error + wantOpcode OpCode + wantVals []uint64 }{ { name: "DUPN", @@ -1069,55 +1071,70 @@ func TestEIP8024_Execution(t *testing.T) { }, }, { - name: "INVALID_SWAPN_LOW", - codeHex: "e75b", - wantErr: true, + name: "INVALID_SWAPN_LOW", + codeHex: "e75b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, }, { name: "JUMP over INVALID_DUPN", codeHex: "600456e65b", - wantErr: false, + wantErr: nil, + }, + { + name: "UNDERFLOW_DUPN_1", + codeHex: "6000808080808080808080808080808080e600", + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, }, // Additional test cases { - name: "INVALID_DUPN_LOW", - codeHex: "e65b", - wantErr: true, + name: "INVALID_DUPN_LOW", + codeHex: "e65b", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, }, { - name: "INVALID_EXCHANGE_LOW", - codeHex: "e850", - wantErr: true, + name: "INVALID_EXCHANGE_LOW", + codeHex: "e850", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, }, { - name: "INVALID_DUPN_HIGH", - codeHex: "e67f", - wantErr: true, + name: "INVALID_DUPN_HIGH", + codeHex: "e67f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: DUPN, }, { - name: "INVALID_SWAPN_HIGH", - codeHex: "e77f", - wantErr: true, + name: "INVALID_SWAPN_HIGH", + codeHex: "e77f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: SWAPN, }, { - name: "INVALID_EXCHANGE_HIGH", - codeHex: "e87f", - wantErr: true, + name: "INVALID_EXCHANGE_HIGH", + codeHex: "e87f", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, }, { - name: "UNDERFLOW_DUPN", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) - wantErr: true, + name: "UNDERFLOW_DUPN_2", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) + wantErr: &ErrStackUnderflow{}, + wantOpcode: DUPN, }, { - name: "UNDERFLOW_SWAPN", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) - wantErr: true, + name: "UNDERFLOW_SWAPN", + codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) + wantErr: &ErrStackUnderflow{}, + wantOpcode: SWAPN, }, { - name: "UNDERFLOW_EXCHANGE", - codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 - wantErr: true, + name: "UNDERFLOW_EXCHANGE", + codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 + wantErr: &ErrStackUnderflow{}, + wantOpcode: EXCHANGE, }, { name: "PC_INCREMENT", @@ -1133,6 +1150,7 @@ func TestEIP8024_Execution(t *testing.T) { pc := uint64(0) scope := &ScopeContext{Stack: stack, Contract: &Contract{Code: code}} var err error + var errOp OpCode for pc < uint64(len(code)) && err == nil { op := code[pc] switch OpCode(op) { @@ -1149,6 +1167,8 @@ func TestEIP8024_Execution(t *testing.T) { _, err = opJumpdest(&pc, evm, scope) case ISZERO: _, err = opIszero(&pc, evm, scope) + case PUSH0: + _, err = opPush0(&pc, evm, scope) case DUPN: _, err = opDupN(&pc, evm, scope) case SWAPN: @@ -1156,14 +1176,37 @@ func TestEIP8024_Execution(t *testing.T) { case EXCHANGE: _, err = opExchange(&pc, evm, scope) default: - err = &ErrInvalidOpCode{opcode: OpCode(op)} + t.Fatalf("unexpected opcode %s at pc=%d", OpCode(op), pc) + } + if err != nil { + errOp = OpCode(op) } pc++ } - if tc.wantErr { + if tc.wantErr != nil { + // Fail because we wanted an error, but didn't get one. if err == nil { t.Fatalf("expected error, got nil") } + // Fail if the wrong opcode threw an error. + if errOp != tc.wantOpcode { + t.Fatalf("expected error from opcode %s, got %s", tc.wantOpcode, errOp) + } + // Fail if we don't get the error we expect. + switch tc.wantErr.(type) { + case *ErrInvalidOpCode: + var want *ErrInvalidOpCode + if !errors.As(err, &want) { + t.Fatalf("expected ErrInvalidOpCode, got %v", err) + } + case *ErrStackUnderflow: + var want *ErrStackUnderflow + if !errors.As(err, &want) { + t.Fatalf("expected ErrStackUnderflow, got %v", err) + } + default: + t.Fatalf("unsupported wantErr type %T", tc.wantErr) + } return } if err != nil { diff --git a/eth/backend.go b/eth/backend.go index aed1542aeb..eaa68b501c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -495,6 +495,9 @@ func (s *Ethereum) updateFilterMapsHeads() { if head == nil || newHead.Hash() != head.Hash() { head = newHead chainView := s.newChainView(head) + if chainView == nil { + return + } historyCutoff, _ := s.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := s.blockchain.CurrentFinalBlock(); fb != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 609c3f4d8b..f76c35a1d5 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -1373,3 +1373,350 @@ func TestStandardTraceBlockToFile(t *testing.T) { } } } + +func TestTraceBadBlock(t *testing.T) { + t.Parallel() + + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + txHashs = make([]common.Hash, 0, 2) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.PUSH1), 0x2a, // push 42 + byte(vm.PUSH1), 0x00, // push slot 0 + byte(vm.SSTORE), // sstore(0, 42) + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + // tx 1: call storage contract (executes PUSH1, PUSH1, SSTORE, STOP) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block so parent state is available + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + api := NewAPI(backend) + result, err := api.TraceBadBlock(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 tx traces, got %d", len(result)) + } + + // First tx: plain transfer + have, _ := json.Marshal(result) + var traces []struct { + TxHash common.Hash `json:"txHash"` + Result struct { + Gas uint64 `json:"gas"` + Failed bool `json:"failed"` + StructLogs []json.RawMessage `json:"structLogs"` + } `json:"result"` + } + if err := json.Unmarshal(have, &traces); err != nil { + t.Fatalf("failed to unmarshal traces: %v", err) + } + if traces[0].TxHash != txHashs[0] { + t.Errorf("tx 0: hash mismatch, have %v, want %v", traces[0].TxHash, txHashs[0]) + } + if traces[0].Result.Gas != params.TxGas { + t.Errorf("tx 0: gas mismatch, have %d, want %d", traces[0].Result.Gas, params.TxGas) + } + if len(traces[0].Result.StructLogs) != 0 { + t.Errorf("tx 0: expected empty structLogs for plain transfer, got %d entries", len(traces[0].Result.StructLogs)) + } + + // Second tx: contract call + if traces[1].TxHash != txHashs[1] { + t.Errorf("tx 1: hash mismatch, have %v, want %v", traces[1].TxHash, txHashs[1]) + } + if traces[1].Result.Failed { + t.Error("tx 1: expected success, got failed") + } + // Contract has 4 opcodes: PUSH1, PUSH1, SSTORE, STOP + if len(traces[1].Result.StructLogs) != 4 { + t.Errorf("tx 1: expected 4 structLog entries for contract call, got %d", len(traces[1].Result.StructLogs)) + } + + // Non-existent bad block + _, err = api.TraceBadBlock(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } + wantErr := fmt.Sprintf("bad block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestIntermediateRoots(t *testing.T) { + t.Parallel() + + // Initialize test accounts and a contract that writes to storage. + var ( + accounts = newAccounts(2) + storageContract = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer = types.HomesteadSigner{} + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + // Contract: SSTORE(CALLVALUE, CALLVALUE) + storageContract: { + Nonce: 1, + Balance: big.NewInt(0), + Code: []byte{ + byte(vm.CALLVALUE), + byte(vm.CALLVALUE), + byte(vm.SSTORE), + byte(vm.STOP), + }, + }, + }, + } + ) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + // tx 0: plain transfer + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 1: sstore(1, 1) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &storageContract, + Value: big.NewInt(1), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + + // tx 2: sstore(2, 2) + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 2, + To: &storageContract, + Value: big.NewInt(2), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + }) + defer backend.teardown() + + api := NewAPI(backend) + block := backend.chain.GetBlockByNumber(1) + + // Should return one root per tx + roots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error, have %v", err) + } + if len(roots) != 3 { + t.Fatalf("root count mismatch, have %d, want 3", len(roots)) + } + for i, root := range roots { + if root == (common.Hash{}) { + t.Errorf("root[%d] should not be zero", i) + } + } + if roots[0] == roots[1] { + t.Error("root[0] and root[1] should differ (transfer vs sstore)") + } + if roots[1] == roots[2] { + t.Error("root[1] and root[2] should differ (sstore to different slots)") + } + + // Intermediate roots of a bad block + rawdb.WriteBadBlock(backend.chaindb, block) + badRoots, err := api.IntermediateRoots(context.Background(), block.Hash(), nil) + if err != nil { + t.Fatalf("want no error for bad block fallback, have %v", err) + } + if !reflect.DeepEqual(roots, badRoots) { + t.Errorf("bad block roots mismatch, have %v, want %v", badRoots, roots) + } + + // Genesis block: should return error + genesisBlock := backend.chain.GetBlockByNumber(0) + _, err = api.IntermediateRoots(context.Background(), genesisBlock.Hash(), nil) + if err == nil || err.Error() != "genesis is not traceable" { + t.Fatalf("want 'genesis is not traceable' error, have %v", err) + } + + // Non-existent block: should return error + _, err = api.IntermediateRoots(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent block, have none") + } + wantErr := fmt.Sprintf("block %#x not found", common.Hash{42}) + if err.Error() != wantErr { + t.Errorf("error mismatch, want '%s', have '%v'", wantErr, err) + } +} + +func TestStandardTraceBadBlockToFile(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + + aa = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f43") + aaCode = []byte{byte(vm.PUSH1), 0x00, byte(vm.POP)} + + bb = common.HexToAddress("0x7217d81b76bdd8707601e959454e3d776aee5f44") + bbCode = []byte{byte(vm.PUSH2), 0x00, 0x01, byte(vm.POP)} + ) + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + aa: { + Code: aaCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + bb: { + Code: bbCode, + Nonce: 1, + Balance: big.NewInt(0), + }, + }, + } + txHashs := make([]common.Hash, 0, 2) + backend := newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: big.NewInt(0), + Gas: 50000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: 1, + To: &bb, + Value: big.NewInt(1), + Gas: 100000, + GasPrice: b.BaseFee(), + Data: nil, + }), types.HomesteadSigner{}, key) + b.AddTx(tx) + txHashs = append(txHashs, tx.Hash()) + }) + defer backend.teardown() + + // Write the block as a bad block + block := backend.chain.GetBlockByNumber(1) + rawdb.WriteBadBlock(backend.chaindb, block) + + var testSuite = []struct { + config *StdTraceConfig + want []string + }{ + { + // All txs traced + config: nil, + want: []string{ + `{"pc":0,"op":96,"gas":"0x7148","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"} +{"pc":2,"op":80,"gas":"0x7145","gasCost":"0x2","memSize":0,"stack":["0x0"],"depth":1,"refund":0,"opName":"POP"} +{"pc":3,"op":0,"gas":"0x7143","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + { + // Specific tx traced + config: &StdTraceConfig{TxHash: txHashs[1]}, + want: []string{ + `{"pc":0,"op":97,"gas":"0x13498","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH2"} +{"pc":3,"op":80,"gas":"0x13495","gasCost":"0x2","memSize":0,"stack":["0x1"],"depth":1,"refund":0,"opName":"POP"} +{"pc":4,"op":0,"gas":"0x13493","gasCost":"0x0","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"STOP"} +{"output":"","gasUsed":"0x5"} +`, + }, + }, + } + + api := NewAPI(backend) + for i, tc := range testSuite { + txTraces, err := api.StandardTraceBadBlockToFile(context.Background(), block.Hash(), tc.config) + if err != nil { + t.Fatalf("test %d: unexpected error %v", i, err) + } + if len(txTraces) != len(tc.want) { + t.Fatalf("test %d: file count mismatch, have %d, want %d", i, len(txTraces), len(tc.want)) + } + for j, traceFileName := range txTraces { + defer os.Remove(traceFileName) + traceReceived, err := os.ReadFile(traceFileName) + if err != nil { + t.Fatalf("test %d: could not read trace file: %v", i, err) + } + if tc.want[j] != string(traceReceived) { + t.Fatalf("test %d, trace %d: result mismatch\nhave:\n%s\nwant:\n%s", i, j, string(traceReceived), tc.want[j]) + } + } + } + + // Non-existent bad block + _, err := api.StandardTraceBadBlockToFile(context.Background(), common.Hash{42}, nil) + if err == nil { + t.Fatal("want error for non-existent bad block, have none") + } +} diff --git a/ethclient/gethclient/gen_callframe_json.go b/ethclient/gethclient/gen_callframe_json.go new file mode 100644 index 0000000000..48df2790cc --- /dev/null +++ b/ethclient/gethclient/gen_callframe_json.go @@ -0,0 +1,104 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallFrame) MarshalJSON() ([]byte, error) { + type CallFrame0 struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input hexutil.Bytes `json:"input"` + Output hexutil.Bytes `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var enc CallFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.RevertReason = c.RevertReason + enc.Calls = c.Calls + enc.Logs = c.Logs + enc.Value = (*hexutil.Big)(c.Value) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallFrame) UnmarshalJSON(input []byte) error { + type CallFrame0 struct { + Type *string `json:"type"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input *hexutil.Bytes `json:"input"` + Output *hexutil.Bytes `json:"output,omitempty"` + Error *string `json:"error,omitempty"` + RevertReason *string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + } + var dec CallFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.RevertReason != nil { + c.RevertReason = *dec.RevertReason + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Logs != nil { + c.Logs = dec.Logs + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/ethclient/gethclient/gen_calllog_json.go b/ethclient/gethclient/gen_calllog_json.go new file mode 100644 index 0000000000..50e25d4bb3 --- /dev/null +++ b/ethclient/gethclient/gen_calllog_json.go @@ -0,0 +1,61 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package gethclient + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*callLogMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c CallLog) MarshalJSON() ([]byte, error) { + type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + Index hexutil.Uint `json:"index"` + Position hexutil.Uint `json:"position"` + } + var enc CallLog + enc.Address = c.Address + enc.Topics = c.Topics + enc.Data = c.Data + enc.Index = hexutil.Uint(c.Index) + enc.Position = hexutil.Uint(c.Position) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *CallLog) UnmarshalJSON(input []byte) error { + type CallLog struct { + Address *common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data *hexutil.Bytes `json:"data"` + Index *hexutil.Uint `json:"index"` + Position *hexutil.Uint `json:"position"` + } + var dec CallLog + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Address != nil { + c.Address = *dec.Address + } + if dec.Topics != nil { + c.Topics = dec.Topics + } + if dec.Data != nil { + c.Data = *dec.Data + } + if dec.Index != nil { + c.Index = uint(*dec.Index) + } + if dec.Position != nil { + c.Position = uint(*dec.Position) + } + return nil +} diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index c2013bca2c..e677e2bb21 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,10 +19,12 @@ package gethclient import ( "context" + "encoding/json" "fmt" "math/big" "runtime" "runtime/debug" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" @@ -229,6 +231,124 @@ func (ec *Client) TraceBlock(ctx context.Context, hash common.Hash, config *trac return result, nil } +// CallTracerConfig configures the call tracer for +// TraceTransactionWithCallTracer and TraceCallWithCallTracer. +type CallTracerConfig struct { + // OnlyTopCall, when true, limits tracing to the main (top-level) call only. + OnlyTopCall bool + // WithLog, when true, includes log emissions in the trace output. + WithLog bool + // Timeout is the maximum duration the tracer may run. + // Zero means the server default (5s). + Timeout time.Duration +} + +//go:generate go run github.com/fjl/gencodec -type CallLog -field-override callLogMarshaling -out gen_calllog_json.go + +// CallLog represents a log emitted during a traced call. +type CallLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data []byte `json:"data"` + Index uint `json:"index"` + Position uint `json:"position"` +} + +type callLogMarshaling struct { + Data hexutil.Bytes + Index hexutil.Uint + Position hexutil.Uint +} + +//go:generate go run github.com/fjl/gencodec -type CallFrame -field-override callFrameMarshaling -out gen_callframe_json.go + +// CallFrame contains the result of a call tracer run. +type CallFrame struct { + Type string `json:"type"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty"` + Input []byte `json:"input"` + Output []byte `json:"output,omitempty"` + Error string `json:"error,omitempty"` + RevertReason string `json:"revertReason,omitempty"` + Calls []CallFrame `json:"calls,omitempty"` + Logs []CallLog `json:"logs,omitempty"` + Value *big.Int `json:"value,omitempty"` +} + +type callFrameMarshaling struct { + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Input hexutil.Bytes + Output hexutil.Bytes + Value *hexutil.Big +} + +// TraceTransactionWithCallTracer traces a transaction with the call tracer +// and returns a typed CallFrame. If config is nil, defaults are used. +func (ec *Client) TraceTransactionWithCallTracer(ctx context.Context, txHash common.Hash, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceTransaction", txHash, callTracerConfig(config)) + if err != nil { + return nil, err + } + return &result, nil +} + +// TraceCallWithCallTracer executes a call with the call tracer and returns +// a typed CallFrame. blockNrOrHash selects the block context for the call. +// overrides specifies state overrides (nil for none), blockOverrides specifies +// block header overrides (nil for none), and config configures the tracer +// (nil for defaults). +func (ec *Client) TraceCallWithCallTracer(ctx context.Context, msg ethereum.CallMsg, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides, config *CallTracerConfig) (*CallFrame, error) { + var result CallFrame + err := ec.c.CallContext(ctx, &result, "debug_traceCall", toCallArg(msg), blockNrOrHash, callTraceCallConfig(config, overrides, blockOverrides)) + if err != nil { + return nil, err + } + return &result, nil +} + +// callTracerConfig converts a CallTracerConfig to the wire-format TraceConfig. +func callTracerConfig(config *CallTracerConfig) *tracers.TraceConfig { + tracer := "callTracer" + tc := &tracers.TraceConfig{Tracer: &tracer} + if config != nil { + if config.OnlyTopCall || config.WithLog { + cfg, _ := json.Marshal(struct { + OnlyTopCall bool `json:"onlyTopCall"` + WithLog bool `json:"withLog"` + }{config.OnlyTopCall, config.WithLog}) + tc.TracerConfig = cfg + } + if config.Timeout != 0 { + s := config.Timeout.String() + tc.Timeout = &s + } + } + return tc +} + +// callTraceCallConfig builds the wire-format TraceCallConfig for debug_traceCall, +// bundling tracer settings with optional state and block overrides. +func callTraceCallConfig(config *CallTracerConfig, overrides map[common.Address]OverrideAccount, blockOverrides *BlockOverrides) interface{} { + tc := callTracerConfig(config) + // debug_traceCall expects a single config object that includes both + // tracer settings and any state/block overrides. + type traceCallConfig struct { + *tracers.TraceConfig + StateOverrides map[common.Address]OverrideAccount `json:"stateOverrides,omitempty"` + BlockOverrides *BlockOverrides `json:"blockOverrides,omitempty"` + } + return &traceCallConfig{ + TraceConfig: tc, + StateOverrides: overrides, + BlockOverrides: blockOverrides, + } +} + func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 0eed63cacf..4d8ccfcb6f 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/tracers" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -161,6 +162,12 @@ func TestGethClient(t *testing.T) { }, { "TestCallContractWithBlockOverrides", func(t *testing.T) { testCallContractWithBlockOverrides(t, client) }, + }, { + "TestTraceTransactionWithCallTracer", + func(t *testing.T) { testTraceTransactionWithCallTracer(t, client, txHashes) }, + }, { + "TestTraceCallWithCallTracer", + func(t *testing.T) { testTraceCallWithCallTracer(t, client) }, }, // The testaccesslist is a bit time-sensitive: the newTestBackend imports // one block. The `testAccessList` fails if the miner has not yet created a @@ -620,3 +627,60 @@ func testCallContractWithBlockOverrides(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected result: %x", res) } } + +func testTraceTransactionWithCallTracer(t *testing.T, client *rpc.Client, txHashes []common.Hash) { + ec := New(client) + for _, txHash := range txHashes { + // With nil config (defaults). + result, err := ec.TraceTransactionWithCallTracer(context.Background(), txHash, nil) + if err != nil { + t.Fatalf("nil config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } + + // With explicit config. + result, err = ec.TraceTransactionWithCallTracer(context.Background(), txHash, + &CallTracerConfig{}, + ) + if err != nil { + t.Fatalf("explicit config: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + } +} + +func testTraceCallWithCallTracer(t *testing.T, client *rpc.Client) { + ec := New(client) + msg := ethereum.CallMsg{ + From: testAddr, + To: &common.Address{}, + Gas: 21000, + GasPrice: big.NewInt(1000000000), + Value: big.NewInt(1), + } + result, err := ec.TraceCallWithCallTracer(context.Background(), msg, + rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), nil, nil, nil, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Type != "CALL" { + t.Fatalf("unexpected type: %s", result.Type) + } + if result.From == (common.Address{}) { + t.Fatal("from is zero") + } + if result.Gas == 0 { + t.Fatal("gas is zero") + } +} diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 395daa6cf4..6b549f40d9 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -307,7 +307,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( // These two settings define the conditions under which compaction concurrency // is increased. Specifically, one additional compaction job will be enabled when: // - there is one more overlapping sub-level0; - // - there is an additional 512 MB of compaction debt; + // - there is an additional 256 MB of compaction debt; // // The maximum concurrency is still capped by MaxConcurrentCompactions, but with // these settings compactions can scale up more readily. diff --git a/go.mod b/go.mod index 306b08ff1a..e15d29a6c5 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 + github.com/klauspost/compress v1.17.8 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 @@ -61,27 +62,35 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.27.5 go.opentelemetry.io/otel v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 go.opentelemetry.io/otel/sdk v1.39.0 go.opentelemetry.io/otel/trace v1.39.0 go.uber.org/automaxprocs v1.5.2 go.uber.org/goleak v1.3.0 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df - golang.org/x/sync v0.12.0 + golang.org/x/sync v0.18.0 golang.org/x/sys v0.39.0 - golang.org/x/text v0.23.0 + golang.org/x/text v0.31.0 golang.org/x/time v0.9.0 - golang.org/x/tools v0.29.0 - google.golang.org/protobuf v1.34.2 + golang.org/x/tools v0.38.0 + google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.77.0 // indirect ) require ( @@ -126,7 +135,6 @@ require ( github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -153,8 +161,8 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index dad819e09d..fe2a64eaec 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -194,6 +196,8 @@ github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasn github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/holiman/billy v0.0.0-20250707135307-f2f9b9aae7db h1:IZUYC/xb3giYwBLMnr8d0TGTzPKFGNTCGgGLoyeX330= @@ -378,6 +382,10 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= @@ -386,6 +394,8 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -398,16 +408,16 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= -golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -423,8 +433,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -433,8 +443,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -486,8 +496,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= @@ -499,21 +509,29 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= -golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b h1:uA40e2M6fYRBf0+8uN5mLlqUtV192iiksiICIBkYJ1E= +google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:Xa7le7qx2vmqB/SzWUBa7KdMjpdpAHlh5QCSnjessQk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/download/download.go b/internal/download/download.go index 26c7795ce5..c59c8a90c3 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -205,7 +205,10 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { if err != nil { return err } - dst := newDownloadWriter(fd, resp.ContentLength) + var dst io.WriteCloser = fd + if resp.ContentLength > 0 { + dst = newDownloadWriter(fd, resp.ContentLength) + } _, err = io.Copy(dst, resp.Body) dst.Close() if err != nil { diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go index 83a761f1fd..72e36fe00f 100644 --- a/internal/era/accumulator.go +++ b/internal/era/accumulator.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "slices" "github.com/ethereum/go-ethereum/common" ssz "github.com/ferranbt/fastssz" @@ -31,8 +32,8 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro if len(hashes) != len(tds) { return common.Hash{}, errors.New("must have equal number hashes as td values") } - if len(hashes) > MaxEra1Size { - return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size) + if len(hashes) > MaxSize { + return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxSize) } hh := ssz.NewHasher() for i := range hashes { @@ -43,7 +44,7 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro } hh.Append(root[:]) } - hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxSize)) return hh.HashRoot() } @@ -69,23 +70,15 @@ func (h *headerRecord) HashTreeRoot() ([32]byte, error) { // HashTreeRootWith ssz hashes the headerRecord object with a hasher. func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { hh.PutBytes(h.Hash[:]) - td := bigToBytes32(h.TotalDifficulty) + td := BigToBytes32(h.TotalDifficulty) hh.PutBytes(td[:]) hh.Merkleize(0) return } // bigToBytes32 converts a big.Int into a little-endian 32-byte array. -func bigToBytes32(n *big.Int) (b [32]byte) { +func BigToBytes32(n *big.Int) (b [32]byte) { n.FillBytes(b[:]) - reverseOrder(b[:]) + slices.Reverse(b[:]) return } - -// reverseOrder reverses the byte order of a slice. -func reverseOrder(b []byte) []byte { - for i := 0; i < 16; i++ { - b[i], b[32-i-1] = b[32-i-1], b[i] - } - return b -} diff --git a/internal/era/era.go b/internal/era/era.go index 118c67abfd..a3c8465bc4 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -1,4 +1,4 @@ -// Copyright 2024 The go-ethereum Authors +// Copyright 2025 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -17,7 +17,6 @@ package era import ( - "encoding/binary" "fmt" "io" "math/big" @@ -25,293 +24,132 @@ import ( "path" "strconv" "strings" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/internal/era/e2store" - "github.com/ethereum/go-ethereum/rlp" - "github.com/golang/snappy" ) +// Type constants for the e2store entries in the Era1 and EraE formats. var ( - TypeVersion uint16 = 0x3265 - TypeCompressedHeader uint16 = 0x03 - TypeCompressedBody uint16 = 0x04 - TypeCompressedReceipts uint16 = 0x05 - TypeTotalDifficulty uint16 = 0x06 - TypeAccumulator uint16 = 0x07 - TypeBlockIndex uint16 = 0x3266 + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeCompressedSlimReceipts uint16 = 0x0a // uses eth/69 encoding + TypeProof uint16 = 0x0b + TypeBlockIndex uint16 = 0x3266 + TypeComponentIndex uint16 = 0x3267 - MaxEra1Size = 8192 + MaxSize = 8192 + // headerSize uint64 = 8 ) -// Filename returns a recognizable Era1-formatted file name for the specified -// epoch and network. -func Filename(network string, epoch int, root common.Hash) string { - return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) -} - -// ReadDir reads all the era1 files in a directory for a given network. -// Format: --.era1 -func ReadDir(dir, network string) ([]string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return nil, fmt.Errorf("error reading directory %s: %w", dir, err) - } - var ( - next = uint64(0) - eras []string - ) - for _, entry := range entries { - if path.Ext(entry.Name()) != ".era1" { - continue - } - parts := strings.Split(entry.Name(), "-") - if len(parts) != 3 || parts[0] != network { - // Invalid era1 filename, skip. - continue - } - if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { - return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) - } else if epoch != next { - return nil, fmt.Errorf("missing epoch %d", next) - } - next += 1 - eras = append(eras, entry.Name()) - } - return eras, nil -} - type ReadAtSeekCloser interface { io.ReaderAt io.Seeker io.Closer } -// Era reads and Era1 file. -type Era struct { - f ReadAtSeekCloser // backing era1 file - s *e2store.Reader // e2store reader over f - m metadata // start, count, length info - mu *sync.Mutex // lock for buf - buf [8]byte // buffer reading entry offsets +// Iterator provides sequential access to blocks in an era file. +type Iterator interface { + // Next advances to the next block. Returns true if a block is available, + // false when iteration is complete or an error occurred. + Next() bool + + // Number returns the block number of the current block. + Number() uint64 + + // Block returns the current block. + Block() (*types.Block, error) + + // BlockAndReceipts returns the current block and its receipts. + BlockAndReceipts() (*types.Block, types.Receipts, error) + + // Receipts returns the receipts for the current block. + Receipts() (types.Receipts, error) + + // Error returns any error encountered during iteration. + Error() error } -// From returns an Era backed by f. -func From(f ReadAtSeekCloser) (*Era, error) { - m, err := readMetadata(f) - if err != nil { - return nil, err - } - return &Era{ - f: f, - s: e2store.NewReader(f), - m: m, - mu: new(sync.Mutex), - }, nil +// Builder constructs era files from blocks and receipts. +// +// Builders handle three epoch types automatically: +// - Pre-merge: all blocks have difficulty > 0, TD is stored for each block +// - Transition: starts pre-merge, ends post-merge; TD stored for all blocks +// - Post-merge: all blocks have difficulty == 0, no TD stored +type Builder interface { + // Add appends a block and its receipts to the era file. + // For pre-merge blocks, td must be provided. + // For post-merge blocks, td should be nil. + Add(block *types.Block, receipts types.Receipts, td *big.Int) error + + // AddRLP appends RLP-encoded block components to the era file. + // For pre-merge blocks, td and difficulty must be provided. + // For post-merge blocks, td and difficulty should be nil. + AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error + + // Finalize writes all collected entries and returns the epoch identifier. + // For Era1 (onedb): returns the accumulator root. + // For EraE (execdb): returns the last block hash. + Finalize() (common.Hash, error) + + // Accumulator returns the accumulator root after Finalize has been called. + // Returns nil for post-merge epochs where no accumulator exists. + Accumulator() *common.Hash } -// Open returns an Era backed by the given filename. -func Open(filename string) (*Era, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - return From(f) +// Era represents the interface for reading era data. +type Era interface { + Close() error + Start() uint64 + Count() uint64 + Iterator() (Iterator, error) + GetBlockByNumber(num uint64) (*types.Block, error) + GetRawBodyByNumber(num uint64) ([]byte, error) + GetRawReceiptsByNumber(num uint64) ([]byte, error) + InitialTD() (*big.Int, error) + Accumulator() (common.Hash, error) } -func (e *Era) Close() error { - return e.f.Close() -} +// ReadDir reads all the era files in a directory for a given network. +// Format: --.erae or --.era1 +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) -// GetBlockByNumber returns the block for the given block number. -func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) if err != nil { - return nil, err + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) } - r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off) - if err != nil { - return nil, err - } - var header types.Header - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - r, _, err = newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - var body types.Body - if err := rlp.Decode(r, &body); err != nil { - return nil, err - } - return types.NewBlockWithHeader(&header).WithBody(body), nil -} - -// GetRawBodyByNumber returns the RLP-encoded body for the given block number. -func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - off, err = e.s.SkipN(off, 1) - if err != nil { - return nil, err - } - r, _, err := newSnappyReader(e.s, TypeCompressedBody, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. -func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { - if e.m.start > num || e.m.start+e.m.count <= num { - return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) - } - off, err := e.readOffset(num) - if err != nil { - return nil, err - } - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - r, _, err := newSnappyReader(e.s, TypeCompressedReceipts, off) - if err != nil { - return nil, err - } - return io.ReadAll(r) -} - -// Accumulator reads the accumulator entry in the Era1 file. -func (e *Era) Accumulator() (common.Hash, error) { - entry, err := e.s.Find(TypeAccumulator) - if err != nil { - return common.Hash{}, err - } - return common.BytesToHash(entry.Value), nil -} - -// InitialTD returns initial total difficulty before the difficulty of the -// first block of the Era1 is applied. -func (e *Era) InitialTD() (*big.Int, error) { var ( - r io.Reader - header types.Header - rawTd []byte - n int64 - off int64 - err error + next = uint64(0) + eras []string + dirType string ) - - // Read first header. - if off, err = e.readOffset(e.m.start); err != nil { - return nil, err + for _, entry := range entries { + ext := path.Ext(entry.Name()) + if ext != ".erae" && ext != ".era1" { + continue + } + if dirType == "" { + dirType = ext + } + parts := strings.Split(entry.Name(), "-") + if len(parts) != 3 || parts[0] != network { + // Invalid era filename, skip. + continue + } + if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era filenames: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + if dirType != ext { + return nil, fmt.Errorf("directory %s contains mixed era file formats: want %s, have %s", dir, dirType, ext) + } + next += 1 + eras = append(eras, entry.Name()) } - if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil { - return nil, err - } - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) - if err != nil { - return nil, err - } - - // Read total difficulty after first block. - if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil { - return nil, err - } - rawTd, err = io.ReadAll(r) - if err != nil { - return nil, err - } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) - return td.Sub(td, header.Difficulty), nil -} - -// Start returns the listed start block. -func (e *Era) Start() uint64 { - return e.m.start -} - -// Count returns the total number of blocks in the Era1. -func (e *Era) Count() uint64 { - return e.m.count -} - -// readOffset reads a specific block's offset from the block index. The value n -// is the absolute block number desired. -func (e *Era) readOffset(n uint64) (int64, error) { - var ( - blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header - firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num - indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes - offOffset = firstIndex + indexOffset // offset of block offset - ) - e.mu.Lock() - defer e.mu.Unlock() - clear(e.buf[:]) - if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { - return 0, err - } - // Since the block offset is relative from the start of the block index record - // we need to add the record offset to it's offset to get the block's absolute - // offset. - return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil -} - -// newSnappyReader returns a snappy.Reader for the e2store entry value at off. -func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { - r, n, err := e.ReaderAt(expectedType, off) - if err != nil { - return nil, 0, err - } - return snappy.NewReader(r), int64(n), err -} - -// metadata wraps the metadata in the block index. -type metadata struct { - start uint64 - count uint64 - length int64 -} - -// readMetadata reads the metadata stored in an Era1 file's block index. -func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { - // Determine length of reader. - if m.length, err = f.Seek(0, io.SeekEnd); err != nil { - return - } - b := make([]byte, 16) - // Read count. It's the last 8 bytes of the file. - if _, err = f.ReadAt(b[:8], m.length-8); err != nil { - return - } - m.count = binary.LittleEndian.Uint64(b) - // Read start. It's at the offset -sizeof(m.count) - - // count*sizeof(indexEntry) - sizeof(m.start) - if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { - return - } - m.start = binary.LittleEndian.Uint64(b[8:]) - return + return eras, nil } diff --git a/internal/era/eradl/eradl.go b/internal/era/eradl/eradl.go index 30bd2bc0d5..375b9ad15b 100644 --- a/internal/era/eradl/eradl.go +++ b/internal/era/eradl/eradl.go @@ -86,8 +86,8 @@ func (l *Loader) DownloadAll(destDir string) error { // DownloadBlockRange fetches the era1 files for the given block range. func (l *Loader) DownloadBlockRange(start, end uint64, destDir string) error { - startEpoch := start / uint64(era.MaxEra1Size) - endEpoch := end / uint64(era.MaxEra1Size) + startEpoch := start / uint64(era.MaxSize) + endEpoch := end / uint64(era.MaxSize) return l.DownloadEpochRange(startEpoch, endEpoch, destDir) } diff --git a/internal/era/execdb/builder.go b/internal/era/execdb/builder.go new file mode 100644 index 0000000000..6246b9caae --- /dev/null +++ b/internal/era/execdb/builder.go @@ -0,0 +1,332 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +// EraE file format specification. +// +// The format can be summarized with the following expression: +// +// eraE := Version | CompressedHeader* | CompressedBody* | CompressedSlimReceipts* | TotalDifficulty* | other-entries* | Accumulator? | ComponentIndex +// +// Each basic element is its own e2store entry: +// +// Version = { type: 0x3265, data: nil } +// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } +// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } +// CompressedSlimReceipts = { type: 0x0a, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } +// TotalDifficulty = { type: 0x06, data: uint256 (header.total_difficulty) } +// AccumulatorRoot = { type: 0x07, data: hash_tree_root(List(HeaderRecord, 8192)) } +// ComponentIndex = { type: 0x3267, data: component-index } +// +// Notes: +// - TotalDifficulty is present for pre-merge and merge transition epochs. +// For pure post-merge epochs, TotalDifficulty is omitted entirely. +// - In merge transition epochs, post-merge blocks store the final total +// difficulty (the TD at which the merge occurred). +// - AccumulatorRoot is only written for pre-merge epochs. +// - HeaderRecord is defined in the Portal Network specification. +// - Proofs (type 0x09) are defined in the spec but not yet supported in this implementation. +// +// ComponentIndex stores relative offsets to each block's components: +// +// component-index := starting-number | indexes | indexes | ... | component-count | count +// indexes := header-offset | body-offset | receipts-offset | td-offset? +// +// All values are little-endian uint64. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in an +// EraE file is also 8192. + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Builder is used to build an EraE e2store file. It collects block entries and +// writes them to the underlying e2store.Writer. +type Builder struct { + w *e2store.Writer + + headers [][]byte + hashes []common.Hash // only pre-merge block hashes, for accumulator + bodies [][]byte + receipts [][]byte + tds []*big.Int + + startNum *uint64 + ttd *big.Int // terminal total difficulty + last common.Hash // hash of last block added + accumulator *common.Hash // accumulator root, set by Finalize (nil for post-merge) + + written uint64 + + buf *bytes.Buffer + snappy *snappy.Writer +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) era.Builder { + return &Builder{ + w: e2store.NewWriter(w), + } +} + +// Add writes a block entry and its receipts into the e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return fmt.Errorf("encode header: %w", err) + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return fmt.Errorf("encode body: %w", err) + } + + rs := make([]*types.SlimReceipt, len(receipts)) + for i, receipt := range receipts { + rs[i] = (*types.SlimReceipt)(receipt) + } + er, err := rlp.EncodeToBytes(rs) + if err != nil { + return fmt.Errorf("encode receipts: %w", err) + } + + return b.AddRLP(eh, eb, er, block.Number().Uint64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file. +// The builder automatically handles transition epochs where both pre and post-merge blocks exist. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error { + if len(b.headers) >= era.MaxSize { + return fmt.Errorf("exceeds max size %d", era.MaxSize) + } + // Set starting block number on first add. + if b.startNum == nil { + b.startNum = new(uint64) + *b.startNum = number + } + + if difficulty == nil { + return fmt.Errorf("invalid block: difficulty is nil") + } + hasDifficulty := difficulty.Sign() > 0 + // Expect td to be nil for post-merge blocks + // and non-nil for pre-merge blocks. + if hasDifficulty != (td != nil) { + return fmt.Errorf("TD and difficulty mismatch: expected both nil or both non-nil") + } + // After the merge, difficulty must be nil. + post := (b.tds == nil && len(b.headers) > 0) || b.ttd != nil + if post && hasDifficulty { + return fmt.Errorf("post-merge epoch: cannot accept total difficulty for block %d", number) + } + + // If this marks the start of the transition, record final total + // difficulty value. + if b.ttd == nil && len(b.tds) > 0 && !hasDifficulty { + b.ttd = new(big.Int).Set(b.tds[len(b.tds)-1]) + } + + // Record block data. + b.headers = append(b.headers, header) + b.bodies = append(b.bodies, body) + b.receipts = append(b.receipts, receipts) + b.last = blockHash + + // Conditionally write the total difficulty and block hashes. + // - Pre-merge: store total difficulty and block hashes. + // - Transition: only store total difficulty. + // - Post-merge: store neither. + if hasDifficulty { + b.hashes = append(b.hashes, blockHash) + b.tds = append(b.tds, new(big.Int).Set(td)) + } else if b.ttd != nil { + b.tds = append(b.tds, new(big.Int).Set(b.ttd)) + } else { + // Post-merge: no TD or block hashes stored. + } + + return nil +} + +// Accumulator returns the accumulator root after Finalize has been called. +// Returns nil for post-merge epochs where no accumulator exists. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + +type offsets struct { + headers []uint64 + bodies []uint64 + receipts []uint64 + tds []uint64 +} + +// Finalize writes all collected block entries to the e2store file. +// For pre-merge or transition epochs, the accumulator root is computed over +// pre-merge blocks and written. For pure post-merge epochs, no accumulator +// is written. Always returns the last block hash as the epoch identifier. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, errors.New("no blocks added, cannot finalize") + } + // Write version before writing any blocks. + if n, err := b.w.Write(era.TypeVersion, nil); err != nil { + return common.Hash{}, fmt.Errorf("write version entry: %w", err) + } else { + b.written += uint64(n) + } + + // Convert TD values to byte-level LE representation. + var tds [][]byte + for _, td := range b.tds { + tds = append(tds, uint256LE(td)) + } + + // Create snappy writer. + b.buf = bytes.NewBuffer(nil) + b.snappy = snappy.NewBufferedWriter(b.buf) + + var o offsets + for _, section := range []struct { + typ uint16 + data [][]byte + compressed bool + offsets *[]uint64 + }{ + {era.TypeCompressedHeader, b.headers, true, &o.headers}, + {era.TypeCompressedBody, b.bodies, true, &o.bodies}, + {era.TypeCompressedSlimReceipts, b.receipts, true, &o.receipts}, + {era.TypeTotalDifficulty, tds, false, &o.tds}, + } { + for _, data := range section.data { + *section.offsets = append(*section.offsets, b.written) + if section.compressed { + // Write snappy compressed data. + if err := b.snappyWrite(section.typ, data); err != nil { + return common.Hash{}, err + } + } else { + // Directly write uncompressed data. + n, err := b.w.Write(section.typ, data) + if err != nil { + return common.Hash{}, err + } + b.written += uint64(n) + } + } + } + + // Compute and write accumulator root only for epochs that started pre-merge. + // The accumulator is computed over only the pre-merge blocks (b.hashes). + // Pure post-merge epochs have no accumulator. + if len(b.tds) > 0 { + accRoot, err := era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)]) + if err != nil { + return common.Hash{}, fmt.Errorf("compute accumulator: %w", err) + } + if n, err := b.w.Write(era.TypeAccumulator, accRoot[:]); err != nil { + return common.Hash{}, fmt.Errorf("write accumulator: %w", err) + } else { + b.written += uint64(n) + } + b.accumulator = &accRoot + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil + } + + // Pure post-merge epoch: no accumulator. + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil +} + +// uin256LE writes 32 byte big integers to little endian. +func uint256LE(v *big.Int) []byte { + b := v.FillBytes(make([]byte, 32)) + for i := 0; i < 16; i++ { + b[i], b[31-i] = b[31-i], b[i] + } + return b +} + +// SnappyWrite compresses the input data using snappy and writes it to the e2store file. +func (b *Builder) snappyWrite(typ uint16, in []byte) error { + b.buf.Reset() + b.snappy.Reset(b.buf) + if _, err := b.snappy.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := b.snappy.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, b.buf.Bytes()) + b.written += uint64(n) + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil +} + +// writeIndex writes the component index to the file. +func (b *Builder) writeIndex(o *offsets) error { + count := len(o.headers) + + // Post-merge, we only index headers, bodies, and receipts. Pre-merge, we also + // need to index the total difficulties. + componentCount := 3 + if len(o.tds) > 0 { + componentCount++ + } + + // Offsets are stored relative to the index position (negative, stored as uint64). + base := int64(b.written) + rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) } + + var buf bytes.Buffer + write := func(v uint64) { binary.Write(&buf, binary.LittleEndian, v) } + + write(*b.startNum) + for i := range o.headers { + write(rel(o.headers[i])) + write(rel(o.bodies[i])) + write(rel(o.receipts[i])) + if len(o.tds) > 0 { + write(rel(o.tds[i])) + } + } + write(uint64(componentCount)) + write(uint64(count)) + + n, err := b.w.Write(era.TypeComponentIndex, buf.Bytes()) + b.written += uint64(n) + return err +} diff --git a/internal/era/execdb/era_test.go b/internal/era/execdb/era_test.go new file mode 100644 index 0000000000..f66931b9ed --- /dev/null +++ b/internal/era/execdb/era_test.go @@ -0,0 +1,348 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "bytes" + "fmt" + "io" + "math/big" + "os" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +func TestEraE(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + start uint64 + preMerge int + postMerge int + accumulator bool // whether accumulator should exist + }{ + { + name: "pre-merge", + start: 0, + preMerge: 128, + postMerge: 0, + accumulator: true, + }, + { + name: "post-merge", + start: 0, + preMerge: 0, + postMerge: 64, + accumulator: false, + }, + { + name: "transition", + start: 0, + preMerge: 32, + postMerge: 32, + accumulator: true, + }, + { + name: "non-zero-start", + start: 8192, + preMerge: 64, + postMerge: 0, + accumulator: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + // Build test data. + type blockData struct { + header, body, receipts []byte + hash common.Hash + td *big.Int + difficulty *big.Int + } + var ( + builder = NewBuilder(f) + blocks []blockData + totalBlocks = tt.preMerge + tt.postMerge + finalTD = big.NewInt(int64(tt.preMerge)) + ) + + // Add pre-merge blocks. + for i := 0; i < tt.preMerge; i++ { + num := tt.start + uint64(i) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(1)}), + body: mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}}), + receipts: mustEncode([]types.SlimReceipt{{CumulativeGasUsed: uint64(i)}}), + hash: common.Hash{byte(i)}, + td: big.NewInt(int64(i + 1)), + difficulty: big.NewInt(1), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, blk.td, blk.difficulty); err != nil { + t.Fatalf("error adding pre-merge block %d: %v", i, err) + } + } + + // Add post-merge blocks. + for i := 0; i < tt.postMerge; i++ { + idx := tt.preMerge + i + num := tt.start + uint64(idx) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(0)}), + body: mustEncode(&types.Body{}), + receipts: mustEncode([]types.SlimReceipt{}), + hash: common.Hash{byte(idx)}, + difficulty: big.NewInt(0), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, nil, big.NewInt(0)); err != nil { + t.Fatalf("error adding post-merge block %d: %v", idx, err) + } + } + + // Finalize and check return values. + epochID, err := builder.Finalize() + if err != nil { + t.Fatalf("error finalizing: %v", err) + } + + // Verify epoch ID is always the last block hash. + expectedLastHash := blocks[len(blocks)-1].hash + if epochID != expectedLastHash { + t.Fatalf("wrong epoch ID: want %s, got %s", expectedLastHash.Hex(), epochID.Hex()) + } + + // Verify accumulator presence. + if tt.accumulator { + if builder.Accumulator() == nil { + t.Fatal("expected non-nil accumulator") + } + } else { + if builder.Accumulator() != nil { + t.Fatalf("expected nil accumulator, got %s", builder.Accumulator().Hex()) + } + } + + // Open and verify the era file. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + // Verify metadata. + if e.Start() != tt.start { + t.Fatalf("wrong start block: want %d, got %d", tt.start, e.Start()) + } + if e.Count() != uint64(totalBlocks) { + t.Fatalf("wrong block count: want %d, got %d", totalBlocks, e.Count()) + } + + // Verify accumulator in file. + if tt.accumulator { + accRoot, err := e.Accumulator() + if err != nil { + t.Fatalf("error getting accumulator: %v", err) + } + if accRoot != *builder.Accumulator() { + t.Fatalf("accumulator mismatch: builder has %s, file contains %s", + builder.Accumulator().Hex(), accRoot.Hex()) + } + } else { + if _, err := e.Accumulator(); err == nil { + t.Fatal("expected error when reading accumulator from post-merge epoch") + } + } + + // Verify blocks via raw iterator. + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %v", err) + } + for i := 0; i < totalBlocks; i++ { + if !it.Next() { + t.Fatalf("expected more entries at %d", i) + } + if it.Error() != nil { + t.Fatalf("unexpected error: %v", it.Error()) + } + + // Check header. + rawHeader, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(rawHeader, blocks[i].header) { + t.Fatalf("mismatched header at %d", i) + } + + // Check body. + rawBody, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(rawBody, blocks[i].body) { + t.Fatalf("mismatched body at %d", i) + } + + // Check receipts. + rawReceipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(rawReceipts, blocks[i].receipts) { + t.Fatalf("mismatched receipts at %d", i) + } + + // Check TD (only for epochs that have TD stored). + if tt.preMerge > 0 && it.TotalDifficulty != nil { + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading TD: %v", err) + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + var expectedTD *big.Int + if i < tt.preMerge { + expectedTD = blocks[i].td + } else { + // Post-merge blocks in transition epoch use final TD. + expectedTD = finalTD + } + if td.Cmp(expectedTD) != 0 { + t.Fatalf("mismatched TD at %d: want %s, got %s", i, expectedTD, td) + } + } + } + + // Verify random access. + for _, blockNum := range []uint64{tt.start, tt.start + uint64(totalBlocks) - 1} { + blk, err := e.GetBlockByNumber(blockNum) + if err != nil { + t.Fatalf("error getting block %d: %v", blockNum, err) + } + if blk.Number().Uint64() != blockNum { + t.Fatalf("wrong block number: want %d, got %d", blockNum, blk.Number().Uint64()) + } + } + + // Verify out-of-range access fails. + if _, err := e.GetBlockByNumber(tt.start + uint64(totalBlocks)); err == nil { + t.Fatal("expected error for out-of-range block") + } + if tt.start > 0 { + if _, err := e.GetBlockByNumber(tt.start - 1); err == nil { + t.Fatal("expected error for block before start") + } + } + + // Verify high-level iterator. + hlIt, err := e.Iterator() + if err != nil { + t.Fatalf("failed to create iterator: %v", err) + } + count := 0 + for hlIt.Next() { + blk, err := hlIt.Block() + if err != nil { + t.Fatalf("error getting block: %v", err) + } + if blk.Number().Uint64() != tt.start+uint64(count) { + t.Fatalf("wrong block number: want %d, got %d", tt.start+uint64(count), blk.Number().Uint64()) + } + count++ + } + if hlIt.Error() != nil { + t.Fatalf("iterator error: %v", hlIt.Error()) + } + if count != totalBlocks { + t.Fatalf("wrong iteration count: want %d, got %d", totalBlocks, count) + } + }) + } +} + +// TestInitialTD tests the InitialTD calculation separately since it requires +// specific TD/difficulty values. +func TestInitialTD(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-initial-td-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + builder := NewBuilder(f) + + // First block: difficulty=5, TD=10, so initial TD = 10-5 = 5. + header := mustEncode(&types.Header{Number: big.NewInt(0), Difficulty: big.NewInt(5)}) + body := mustEncode(&types.Body{}) + receipts := mustEncode([]types.SlimReceipt{}) + + if err := builder.AddRLP(header, body, receipts, 0, common.Hash{0}, big.NewInt(10), big.NewInt(5)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + // Second block: difficulty=3, TD=13. + header2 := mustEncode(&types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(3)}) + if err := builder.AddRLP(header2, body, receipts, 1, common.Hash{1}, big.NewInt(13), big.NewInt(3)); err != nil { + t.Fatalf("error adding block: %v", err) + } + + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing: %v", err) + } + + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + initialTD, err := e.InitialTD() + if err != nil { + t.Fatalf("error getting initial TD: %v", err) + } + + // Initial TD should be TD[0] - Difficulty[0] = 10 - 5 = 5. + if initialTD.Cmp(big.NewInt(5)) != 0 { + t.Fatalf("wrong initial TD: want 5, got %s", initialTD) + } +} + +func mustEncode(obj any) []byte { + b, err := rlp.EncodeToBytes(obj) + if err != nil { + panic(fmt.Sprintf("failed to encode obj: %v", err)) + } + return b +} diff --git a/internal/era/execdb/iterator.go b/internal/era/execdb/iterator.go new file mode 100644 index 0000000000..8d17ac00a9 --- /dev/null +++ b/internal/era/execdb/iterator.go @@ -0,0 +1,240 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "errors" + "io" + "math/big" + "slices" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +type Iterator struct { + inner *RawIterator + block *types.Block // cache for decoded block +} + +// NewIterator returns a header/body/receipt iterator over the archive. +// Call Next immediately to position on the first block. +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) + if err != nil { + return nil, err + } + return &Iterator{inner: inner}, nil +} + +// Next advances to the next block entry. +func (it *Iterator) Next() bool { + it.block = nil + return it.inner.Next() +} + +// Number is the number of the block currently loaded. +func (it *Iterator) Number() uint64 { return it.inner.next - 1 } + +// Error returns any iteration error (EOF is reported as nil, identical +// to the Era‑1 iterator behaviour). +func (it *Iterator) Error() error { return it.inner.Error() } + +// Block decodes the current header+body into a *types.Block. +func (it *Iterator) Block() (*types.Block, error) { + if it.block != nil { + return it.block, nil + } + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non‑nil") + } + var ( + h types.Header + b types.Body + ) + if err := rlp.Decode(it.inner.Header, &h); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &b); err != nil { + return nil, err + } + it.block = types.NewBlockWithHeader(&h).WithBody(b) + return it.block, nil +} + +// Receipts decodes receipts for the current block. +func (it *Iterator) Receipts() (types.Receipts, error) { + block, err := it.Block() + if err != nil { + return nil, err + } + if it.inner.Receipts == nil { + return nil, errors.New("receipts must be non‑nil") + } + var rs []*types.SlimReceipt + if err := rlp.Decode(it.inner.Receipts, &rs); err != nil { + return nil, err + } + if len(rs) != len(block.Transactions()) { + return nil, errors.New("number of txs does not match receipts") + } + receipts := make([]*types.Receipt, len(rs)) + for i, receipt := range rs { + receipts[i] = (*types.Receipt)(receipt) + receipts[i].Bloom = types.CreateBloom(receipts[i]) + } + return receipts, nil +} + +// BlockAndReceipts is a convenience wrapper. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the TD at the current position (if present). +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + if it.inner.TotalDifficulty == nil { + return nil, errors.New("total‑difficulty stream is nil") + } + tdBytes, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + slices.Reverse(tdBytes) + return new(big.Int).SetBytes(tdBytes), nil +} + +// ----------------------------------------------------------------------------- +// Low‑level iterator (raw TLV/offset handling, no decoding) +// ----------------------------------------------------------------------------- + +type RawIterator struct { + e *Era + next uint64 // next block to pull + err error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader // nil when archive omits TD +} + +// NewRawIterator creates an iterator positioned *before* the first block. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{e: e, next: e.m.start}, nil +} + +// Next loads the next block’s components; returns false on EOF or error. +func (it *RawIterator) Next() bool { + it.err = nil // clear previous error + + if it.next >= it.e.m.start+it.e.m.count { + it.clear() + return false + } + + headerOffset, err := it.e.headerOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Header, _, err = newSnappyReader(it.e.s, era.TypeCompressedHeader, headerOffset) + if err != nil { + it.setErr(err) + return false + } + + bodyOffset, err := it.e.bodyOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Body, _, err = newSnappyReader(it.e.s, era.TypeCompressedBody, bodyOffset) + if err != nil { + it.setErr(err) + return false + } + + receiptsOffset, err := it.e.receiptOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedSlimReceipts, receiptsOffset) + if err != nil { + it.setErr(err) + return false + } + + // Check if TD component is present in this file (pre-merge or merge-transition epoch). + if int(td) < int(it.e.m.components) { + tdOffset, err := it.e.tdOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.TotalDifficulty, _, err = it.e.s.ReaderAt(era.TypeTotalDifficulty, tdOffset) + if err != nil { + it.setErr(err) + return false + } + } else { + it.TotalDifficulty = nil + } + + it.next++ + return true +} + +func (it *RawIterator) Number() uint64 { return it.next - 1 } + +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +func (it *RawIterator) setErr(err error) { + it.err = err + it.clear() +} + +func (it *RawIterator) clear() { + it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil +} + +// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader +// plus the length of the underlying TLV payload so callers can advance offsets. +func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) { + raw, n, err := r.ReaderAt(typ, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(raw), int64(n), nil +} diff --git a/internal/era/execdb/reader.go b/internal/era/execdb/reader.go new file mode 100644 index 0000000000..d0aaad1748 --- /dev/null +++ b/internal/era/execdb/reader.go @@ -0,0 +1,296 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package execdb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +// Era object represents an era file that contains blocks and their components. +type Era struct { + f era.ReadAtSeekCloser + s *e2store.Reader + m metadata // metadata for the Era file +} + +// Filename returns a recognizable filename for an EraE file. +// The filename uses the last block hash to uniquely identify the epoch's content. +func Filename(network string, epoch int, lastBlockHash common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.erae", network, epoch, lastBlockHash.Hex()[2:10]) +} + +// Open accesses the era file. +func Open(path string) (*Era, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Close closes the era file safely. +func (e *Era) Close() error { + if e.f == nil { + return nil + } + err := e.f.Close() + e.f = nil + return err +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// Start retrieves the starting block number. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count retrieves the count of blocks present. +func (e *Era) Count() uint64 { + return e.m.count +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber retrieves the block if present within the era file. +func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) { + h, err := e.GetHeader(blockNum) + if err != nil { + return nil, err + } + b, err := e.GetBody(blockNum) + if err != nil { + return nil, err + } + return types.NewBlockWithHeader(h).WithBody(*b), nil +} + +// GetHeader retrieves the header from the era file through the cached offset table. +func (e *Era) GetHeader(num uint64) (*types.Header, error) { + off, err := e.headerOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var h types.Header + return &h, rlp.Decode(r, &h) +} + +// GetBody retrieves the body from the era file through cached offset table. +func (e *Era) GetBody(num uint64) (*types.Body, error) { + off, err := e.bodyOff(num) + if err != nil { + return nil, err + } + + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + + r = snappy.NewReader(r) + var b types.Body + return &b, rlp.Decode(r, &b) +} + +// GetTD retrieves the td from the era file through cached offset table. +func (e *Era) GetTD(blockNum uint64) (*big.Int, error) { + off, err := e.tdOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeTotalDifficulty, off) + if err != nil { + return nil, err + } + buf, _ := io.ReadAll(r) + slices.Reverse(buf) + td := new(big.Int).SetBytes(buf) + return td, nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(blockNum uint64) ([]byte, error) { + off, err := e.bodyOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(blockNum uint64) ([]byte, error) { + off, err := e.receiptOff(blockNum) + if err != nil { + return nil, err + } + r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, off) + if err != nil { + return nil, err + } + r = snappy.NewReader(r) + return io.ReadAll(r) +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era is applied. Returns an error if TD is not available +// (e.g., post-merge epoch). +func (e *Era) InitialTD() (*big.Int, error) { + // Check if TD component exists. + if int(td) >= int(e.m.components) { + return nil, fmt.Errorf("total difficulty not available in this epoch") + } + + // Get first header to read its difficulty. + header, err := e.GetHeader(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first header: %w", err) + } + + // Get TD after first block using the index. + firstTD, err := e.GetTD(e.m.start) + if err != nil { + return nil, fmt.Errorf("read first TD: %w", err) + } + + // Initial TD = TD[0] - Difficulty[0] + return new(big.Int).Sub(firstTD, header.Difficulty), nil +} + +// Accumulator reads the accumulator entry in the EraE file if it exists. +// Note that one premerge erae files will contain an accumulator entry. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// loadIndex loads in the index table containing all offsets and caches it. +func (e *Era) loadIndex() error { + var err error + e.m.length, err = e.f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + + b := make([]byte, 16) + if _, err = e.f.ReadAt(b, e.m.length-16); err != nil { + return err + } + e.m.components = binary.LittleEndian.Uint64(b[0:8]) + e.m.count = binary.LittleEndian.Uint64(b[8:16]) + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + tlvstart := e.m.length - int64(payloadlen) - 8 + _, err = e.f.ReadAt(b[:8], tlvstart+8) + if err != nil { + return err + } + + e.m.start = binary.LittleEndian.Uint64(b[:8]) + return nil +} + +// headerOff, bodyOff, receiptOff, and tdOff return the offsets of the respective components for a given block number. +func (e *Era) headerOff(num uint64) (int64, error) { return e.indexOffset(num, header) } +func (e *Era) bodyOff(num uint64) (int64, error) { return e.indexOffset(num, body) } +func (e *Era) receiptOff(num uint64) (int64, error) { return e.indexOffset(num, receipts) } +func (e *Era) tdOff(num uint64) (int64, error) { return e.indexOffset(num, td) } + +// indexOffset calculates offset to a certain component for a block number within a file. +func (e *Era) indexOffset(n uint64, component componentType) (int64, error) { + if n < e.m.start || n >= e.m.start+e.m.count { + return 0, fmt.Errorf("block %d out of range [%d,%d)", n, e.m.start, e.m.start+e.m.count) + } + if int(component) >= int(e.m.components) { + return 0, fmt.Errorf("component %d not present", component) + } + + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + indstart := e.m.length - int64(payloadlen) - 8 + + rec := (n-e.m.start)*e.m.components + uint64(component) + pos := indstart + 8 + 8 + int64(rec*8) + + var buf [8]byte + if _, err := e.f.ReadAt(buf[:], pos); err != nil { + return 0, err + } + rel := binary.LittleEndian.Uint64(buf[:]) + return int64(rel) + indstart, nil +} + +// metadata contains the information about the era file that is written into the file. +type metadata struct { + start uint64 // start block number + count uint64 // number of blocks in the era + components uint64 // number of properties + length int64 // length of the file in bytes +} + +// componentType represents the integer form of a specific type that can be present in the era file. +type componentType int + +// header, body, receipts, td, and proof are the different types of components that can be present in the era file. +const ( + header componentType = iota + body + receipts + td + proof +) diff --git a/internal/era/builder.go b/internal/era/onedb/builder.go similarity index 84% rename from internal/era/builder.go rename to internal/era/onedb/builder.go index 975561564c..6497bf4f60 100644 --- a/internal/era/builder.go +++ b/internal/era/onedb/builder.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/e2store" "github.com/ethereum/go-ethereum/rlp" "github.com/golang/snappy" @@ -72,20 +73,22 @@ import ( // Due to the accumulator size limit of 8192, the maximum number of blocks in // an Era1 batch is also 8192. type Builder struct { - w *e2store.Writer - startNum *uint64 - startTd *big.Int - indexes []uint64 - hashes []common.Hash - tds []*big.Int - written int + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + accumulator *common.Hash // accumulator root, set by Finalize + + written int buf *bytes.Buffer snappy *snappy.Writer } // NewBuilder returns a new Builder instance. -func NewBuilder(w io.Writer) *Builder { +func NewBuilder(w io.Writer) era.Builder { buf := bytes.NewBuffer(nil) return &Builder{ w: e2store.NewWriter(w), @@ -117,7 +120,7 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { // Write Era1 version entry before first block. if b.startNum == nil { - n, err := b.w.Write(TypeVersion, nil) + n, err := b.w.Write(era.TypeVersion, nil) if err != nil { return err } @@ -126,8 +129,8 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.startTd = new(big.Int).Sub(td, difficulty) b.written += n } - if len(b.indexes) >= MaxEra1Size { - return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + if len(b.indexes) >= era.MaxSize { + return fmt.Errorf("exceeds maximum batch size of %d", era.MaxSize) } b.indexes = append(b.indexes, uint64(b.written)) @@ -135,19 +138,19 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm b.tds = append(b.tds, td) // Write block data. - if err := b.snappyWrite(TypeCompressedHeader, header); err != nil { + if err := b.snappyWrite(era.TypeCompressedHeader, header); err != nil { return err } - if err := b.snappyWrite(TypeCompressedBody, body); err != nil { + if err := b.snappyWrite(era.TypeCompressedBody, body); err != nil { return err } - if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil { + if err := b.snappyWrite(era.TypeCompressedReceipts, receipts); err != nil { return err } // Also write total difficulty, but don't snappy encode. - btd := bigToBytes32(td) - n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + btd := era.BigToBytes32(td) + n, err := b.w.Write(era.TypeTotalDifficulty, btd[:]) b.written += n if err != nil { return err @@ -157,21 +160,24 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm } // Finalize computes the accumulator and block index values, then writes the -// corresponding e2store entries. +// corresponding e2store entries. Era1 always has an accumulator, so this +// always returns a valid hash. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("finalize called on empty builder") } // Compute accumulator root and write entry. - root, err := ComputeAccumulator(b.hashes, b.tds) + root, err := era.ComputeAccumulator(b.hashes, b.tds) if err != nil { return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) } - n, err := b.w.Write(TypeAccumulator, root[:]) + n, err := b.w.Write(era.TypeAccumulator, root[:]) b.written += n if err != nil { return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) } + b.accumulator = &root + // Get beginning of index entry to calculate block relative offset. base := int64(b.written) @@ -196,13 +202,19 @@ func (b *Builder) Finalize() (common.Hash, error) { binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) // Finally, write the block index entry. - if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + if _, err := b.w.Write(era.TypeBlockIndex, index); err != nil { return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) } return root, nil } +// Accumulator returns the accumulator root after Finalize has been called. +// For Era1, this always returns a non-nil value since all blocks are pre-merge. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + // snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. func (b *Builder) snappyWrite(typ uint16, in []byte) error { var ( diff --git a/internal/era/era_test.go b/internal/era/onedb/builder_test.go similarity index 95% rename from internal/era/era_test.go rename to internal/era/onedb/builder_test.go index 31fa0076a6..bc7a1d9e63 100644 --- a/internal/era/era_test.go +++ b/internal/era/onedb/builder_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "bytes" @@ -22,6 +22,7 @@ import ( "io" "math/big" "os" + "slices" "testing" "github.com/ethereum/go-ethereum/common" @@ -82,7 +83,11 @@ func TestEra1Builder(t *testing.T) { t.Fatalf("failed to open era: %v", err) } defer e.Close() - it, err := NewRawIterator(e) + eraPtr, ok := e.(*Era) + if !ok { + t.Fatalf("failed to assert *Era type") + } + it, err := NewRawIterator(eraPtr) if err != nil { t.Fatalf("failed to make iterator: %s", err) } @@ -119,7 +124,7 @@ func TestEra1Builder(t *testing.T) { if !bytes.Equal(rawReceipts, chain.receipts[i]) { t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts) } - receipts, err := getReceiptsByNumber(e, i) + receipts, err := getReceiptsByNumber(eraPtr, i) if err != nil { t.Fatalf("error reading receipts: %v", err) } @@ -136,7 +141,8 @@ func TestEra1Builder(t *testing.T) { if err != nil { t.Fatalf("error reading td: %v", err) } - td := new(big.Int).SetBytes(reverseOrder(rawTd)) + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) if td.Cmp(chain.tds[i]) != 0 { t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) } diff --git a/internal/era/iterator.go b/internal/era/onedb/iterator.go similarity index 89% rename from internal/era/iterator.go rename to internal/era/onedb/iterator.go index 3c4f82d850..b80fbabbc5 100644 --- a/internal/era/iterator.go +++ b/internal/era/onedb/iterator.go @@ -14,14 +14,16 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package era +package onedb import ( "errors" "io" "math/big" + "slices" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/rlp" ) @@ -32,8 +34,8 @@ type Iterator struct { // NewIterator returns a new Iterator instance. Next must be immediately // called on new iterators to load the first item. -func NewIterator(e *Era) (*Iterator, error) { - inner, err := NewRawIterator(e) +func NewIterator(e era.Era) (era.Iterator, error) { + inner, err := NewRawIterator(e.(*Era)) if err != nil { return nil, err } @@ -107,7 +109,8 @@ func (it *Iterator) TotalDifficulty() (*big.Int, error) { if err != nil { return nil, err } - return new(big.Int).SetBytes(reverseOrder(td)), nil + slices.Reverse(td) + return new(big.Int).SetBytes(td), nil } // RawIterator reads an RLP-encode Era1 entries. @@ -151,22 +154,22 @@ func (it *RawIterator) Next() bool { return false } var n int64 - if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + if it.Header, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedHeader, off); it.err != nil { it.clear() return true } off += n - if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + if it.Body, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedBody, off); it.err != nil { it.clear() return true } off += n - if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + if it.Receipts, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, off); it.err != nil { it.clear() return true } off += n - if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(era.TypeTotalDifficulty, off); it.err != nil { it.clear() return true } diff --git a/internal/era/onedb/reader.go b/internal/era/onedb/reader.go new file mode 100644 index 0000000000..df93ca8211 --- /dev/null +++ b/internal/era/onedb/reader.go @@ -0,0 +1,279 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package onedb + +import ( + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +type ReadAtSeekCloser interface { + io.ReaderAt + io.Seeker + io.Closer +} + +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f era.ReadAtSeekCloser) (era.Era, error) { + m, err := readMetadata(f) + if err != nil { + return nil, err + } + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), + }, nil +} + +// Open returns an Era backed by the given filename. +func Open(filename string) (era.Era, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + return From(f) +} + +func (e *Era) Close() error { + return e.f.Close() +} + +// Iterator returns an iterator over the era file. +func (e *Era) Iterator() (era.Iterator, error) { + return NewIterator(e) +} + +// GetBlockByNumber returns the block for the given block number. +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + r, n, err := newSnappyReader(e.s, era.TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + r, _, err = newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.Decode(r, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body), nil +} + +// GetRawBodyByNumber returns the RLP-encoded body for the given block number. +func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + off, err = e.s.SkipN(off, 1) + if err != nil { + return nil, err + } + r, _, err := newSnappyReader(e.s, era.TypeCompressedBody, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number. +func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count) + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + r, _, err := newSnappyReader(e.s, era.TypeCompressedReceipts, off) + if err != nil { + return nil, err + } + return io.ReadAll(r) +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(era.TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int64 + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { + return nil, err + } + if r, n, err = newSnappyReader(e.s, era.TypeCompressedHeader, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + off += n + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + // Read total difficulty after first block. + if r, _, err = e.s.ReaderAt(era.TypeTotalDifficulty, off); err != nil { + return nil, err + } + rawTd, err = io.ReadAll(r) + if err != nil { + return nil, err + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + return td.Sub(td, header.Difficulty), nil +} + +// Start returns the listed start block. +func (e *Era) Start() uint64 { + return e.m.start +} + +// Count returns the total number of blocks in the Era1. +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header + firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clear(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from the start of the block index record + // we need to add the record offset to it's offset to get the block's absolute + // offset. + return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newSnappyReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) { + r, n, err := e.ReaderAt(expectedType, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(r), int64(n), err +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start uint64 + count uint64 + length int64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } + b := make([]byte, 16) + // Read count. It's the last 8 bytes of the file. + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { + return + } + m.count = binary.LittleEndian.Uint64(b) + // Read start. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { + return + } + m.start = binary.LittleEndian.Uint64(b[8:]) + return +} diff --git a/internal/era/proof.go b/internal/era/proof.go new file mode 100644 index 0000000000..464d9e6fe5 --- /dev/null +++ b/internal/era/proof.go @@ -0,0 +1,36 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package era + +import ( + "io" + + "github.com/ethereum/go-ethereum/rlp" +) + +type ProofVariant uint16 + +const ( + ProofNone ProofVariant = iota +) + +// Proof is the interface for all block proof types in the package. +// It's a stub for later integration into Era. +type Proof interface { + EncodeRLP(w io.Writer) error + DecodeRLP(s *rlp.Stream) error + Variant() ProofVariant +} diff --git a/internal/ethapi/errors.go b/internal/ethapi/errors.go index 30711a0167..e406c36d6c 100644 --- a/internal/ethapi/errors.go +++ b/internal/ethapi/errors.go @@ -112,7 +112,6 @@ const ( errCodeClientLimitExceeded = -38026 errCodeInternalError = -32603 errCodeInvalidParams = -32602 - errCodeReverted = -32000 errCodeVMError = -32015 errCodeTxSyncTimeout = 4 ) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index df7827acd6..6fa34a8695 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -310,7 +310,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if errors.Is(result.Err, vm.ErrExecutionReverted) { // If the result contains a revert reason, try to unpack it. revertErr := newRevertError(result.Revert()) - callRes.Error = &callError{Message: revertErr.Error(), Code: errCodeReverted, Data: revertErr.ErrorData().(string)} + callRes.Error = &callError{Message: revertErr.Error(), Code: revertErr.ErrorCode(), Data: revertErr.ErrorData().(string)} } else { callRes.Error = &callError{Message: result.Err.Error(), Code: errCodeVMError} } diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 6bd16da66c..470bafed25 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -46,12 +46,12 @@ func BoolAttribute(key string, val bool) Attribute { } // StartSpan creates a SpanKind=INTERNAL span. -func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(error)) { +func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { return StartSpanWithTracer(ctx, otel.Tracer(""), spanName, attributes...) } // StartSpanWithTracer requires a tracer to be passed in and creates a SpanKind=INTERNAL span. -func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(error)) { +func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { return startSpan(ctx, tracer, trace.SpanKindInternal, name, attributes...) } @@ -67,7 +67,7 @@ type RPCInfo struct { // The span name is formatted as $rpcSystem.$rpcService/$rpcMethod // (e.g. "jsonrpc.engine/newPayloadV4") which follows the Open Telemetry // semantic convensions: https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#span-name. -func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, others ...Attribute) (context.Context, func(error)) { +func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, others ...Attribute) (context.Context, func(*error)) { var ( name = fmt.Sprintf("%s.%s/%s", rpc.System, rpc.Service, rpc.Method) attributes = append([]Attribute{ @@ -84,7 +84,7 @@ func StartServerSpan(ctx context.Context, tracer trace.Tracer, rpc RPCInfo, othe } // startSpan creates a span with the given kind. -func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(error)) { +func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, spanName string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { ctx, span := tracer.Start(ctx, spanName, trace.WithSpanKind(kind)) if len(attributes) > 0 { span.SetAttributes(attributes...) @@ -93,11 +93,11 @@ func startSpan(ctx context.Context, tracer trace.Tracer, kind trace.SpanKind, sp } // endSpan ends the span and handles error recording. -func endSpan(span trace.Span) func(error) { - return func(err error) { - if err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) +func endSpan(span trace.Span) func(*error) { + return func(err *error) { + if err != nil && *err != nil { + span.RecordError(*err) + span.SetStatus(codes.Error, (*err).Error()) } span.End() } diff --git a/internal/telemetry/tracesetup/setup.go b/internal/telemetry/tracesetup/setup.go new file mode 100644 index 0000000000..9637ca1a9b --- /dev/null +++ b/internal/telemetry/tracesetup/setup.go @@ -0,0 +1,162 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracesetup + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "strings" + "time" + + "github.com/ethereum/go-ethereum/internal/version" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.38.0" +) + +const startStopTimeout = 10 * time.Second + +// Service wraps the provider to implement node.Lifecycle. +type Service struct { + endpoint string + exporter *otlptrace.Exporter + provider *sdktrace.TracerProvider +} + +// Start implements node.Lifecycle. +func (t *Service) Start() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.exporter.Start(ctx); err != nil { + log.Error("OpenTelemetry exporter didn't start", "endpoint", t.endpoint, "err", err) + return err + } + log.Info("OpenTelemetry trace export enabled", "endpoint", t.endpoint) + return nil +} + +// Stop implements node.Lifecycle. +func (t *Service) Stop() error { + ctx, cancel := context.WithTimeout(context.Background(), startStopTimeout) + defer cancel() + if err := t.provider.Shutdown(ctx); err != nil { + log.Error("Failed to stop OpenTelemetry service", "err", err) + return err + } + log.Debug("OpenTelemetry stopped") + return nil +} + +// SetupTelemetry initializes telemetry with the given parameters. +func SetupTelemetry(cfg node.OpenTelemetryConfig, stack *node.Node) error { + if !cfg.Enabled { + return nil + } + if cfg.SampleRatio < 0 || cfg.SampleRatio > 1 { + return fmt.Errorf("invalid sample ratio: %f", cfg.SampleRatio) + } + // Create exporter based on endpoint URL + u, err := url.Parse(cfg.Endpoint) + if err != nil { + return fmt.Errorf("invalid rpc tracing endpoint URL: %w", err) + } + var exporter *otlptrace.Exporter + switch u.Scheme { + case "http", "https": + opts := []otlptracehttp.Option{ + otlptracehttp.WithEndpoint(u.Host), + } + if u.Scheme == "http" { + opts = append(opts, otlptracehttp.WithInsecure()) + } + if u.Path != "" && u.Path != "/" { + opts = append(opts, otlptracehttp.WithURLPath(u.Path)) + } + if cfg.AuthUser != "" { + opts = append(opts, otlptracehttp.WithHeaders(map[string]string{ + "Authorization": "Basic " + base64.StdEncoding.EncodeToString([]byte(cfg.AuthUser+":"+cfg.AuthPassword)), + })) + } + exporter = otlptracehttp.NewUnstarted(opts...) + default: + return fmt.Errorf("unsupported telemetry url scheme: %s", u.Scheme) + } + + // Define sampler such that if no parent span is available, + // then sampleRatio of traces are sampled; otherwise, inherit + // the parent's sampling decision. + sampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio)) + + // Define batch span processor options + batchOpts := []sdktrace.BatchSpanProcessorOption{ + // The maximum number of spans that can be queued before dropping + sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxExportBatchSize), + // The maximum number of spans to export in a single batch + sdktrace.WithMaxExportBatchSize(sdktrace.DefaultMaxExportBatchSize), + // How long an export operation can take before timing out + sdktrace.WithExportTimeout(time.Duration(sdktrace.DefaultExportTimeout) * time.Millisecond), + // How often to export, even if the batch isn't full + sdktrace.WithBatchTimeout(time.Duration(sdktrace.DefaultScheduleDelay) * time.Millisecond), + } + + // Define resource attributes + var attr = []attribute.KeyValue{ + semconv.ServiceName("geth"), + attribute.String("client.name", version.ClientName("geth")), + } + // Add instance ID if provided + if cfg.InstanceID != "" { + attr = append(attr, semconv.ServiceInstanceID(cfg.InstanceID)) + } + // Add custom tags if provided + if cfg.Tags != "" { + for tag := range strings.SplitSeq(cfg.Tags, ",") { + key, value, ok := strings.Cut(tag, "=") + if ok { + attr = append(attr, attribute.String(key, value)) + } + } + } + res := resource.NewWithAttributes(semconv.SchemaURL, attr...) + + // Configure TracerProvider and set it as the global tracer provider + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sampler), + sdktrace.WithBatcher(exporter, batchOpts...), + sdktrace.WithResource(res), + ) + otel.SetTracerProvider(tp) + + // Set global propagator for context propagation + // Note: This is needed for distributed tracing + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + service := &Service{endpoint: cfg.Endpoint, exporter: exporter, provider: tp} + stack.RegisterLifecycle(service) + return nil +} diff --git a/metrics/counter.go b/metrics/counter.go index c884e9a178..0e4d93bfbc 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -7,7 +7,10 @@ import ( // GetOrRegisterCounter returns an existing Counter or constructs and registers // a new Counter. func GetOrRegisterCounter(name string, r Registry) *Counter { - return getOrRegister(name, NewCounter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounter() }).(*Counter) } // NewCounter constructs a new Counter. diff --git a/metrics/counter_float64.go b/metrics/counter_float64.go index 6cc73d89a2..caaaeb7be7 100644 --- a/metrics/counter_float64.go +++ b/metrics/counter_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterCounterFloat64 returns an existing *CounterFloat64 or constructs and registers // a new CounterFloat64. func GetOrRegisterCounterFloat64(name string, r Registry) *CounterFloat64 { - return getOrRegister(name, NewCounterFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewCounterFloat64() }).(*CounterFloat64) } // NewCounterFloat64 constructs a new CounterFloat64. diff --git a/metrics/gauge.go b/metrics/gauge.go index 20de95255b..39d5ccb3bc 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -11,7 +11,10 @@ func (g GaugeSnapshot) Value() int64 { return int64(g) } // GetOrRegisterGauge returns an existing Gauge or constructs and registers a // new Gauge. func GetOrRegisterGauge(name string, r Registry) *Gauge { - return getOrRegister(name, NewGauge, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGauge() }).(*Gauge) } // NewGauge constructs a new Gauge. diff --git a/metrics/gauge_float64.go b/metrics/gauge_float64.go index 48524e4c3f..c7a1df0a23 100644 --- a/metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a // new GaugeFloat64. func GetOrRegisterGaugeFloat64(name string, r Registry) *GaugeFloat64 { - return getOrRegister(name, NewGaugeFloat64, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeFloat64() }).(*GaugeFloat64) } // GaugeFloat64Snapshot is a read-only copy of a GaugeFloat64. diff --git a/metrics/gauge_info.go b/metrics/gauge_info.go index 34ac917919..30c5114085 100644 --- a/metrics/gauge_info.go +++ b/metrics/gauge_info.go @@ -16,7 +16,10 @@ func (val GaugeInfoValue) String() string { // GetOrRegisterGaugeInfo returns an existing GaugeInfo or constructs and registers a // new GaugeInfo. func GetOrRegisterGaugeInfo(name string, r Registry) *GaugeInfo { - return getOrRegister(name, NewGaugeInfo, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewGaugeInfo() }).(*GaugeInfo) } // NewGaugeInfo constructs a new GaugeInfo. diff --git a/metrics/histogram.go b/metrics/histogram.go index 18bf6e3d2b..467457abdb 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -23,13 +23,19 @@ type Histogram interface { // GetOrRegisterHistogram returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s) }).(Histogram) } // GetOrRegisterHistogramLazy returns an existing Histogram or constructs and // registers a new StandardHistogram. func GetOrRegisterHistogramLazy(name string, r Registry, s func() Sample) Histogram { - return getOrRegister(name, func() Histogram { return NewHistogram(s()) }, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewHistogram(s()) }).(Histogram) } // NewHistogram constructs a new StandardHistogram from a Sample. diff --git a/metrics/meter.go b/metrics/meter.go index ee23af10eb..2829774d8c 100644 --- a/metrics/meter.go +++ b/metrics/meter.go @@ -12,7 +12,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) *Meter { - return getOrRegister(name, NewMeter, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewMeter() }).(*Meter) } // NewMeter constructs a new Meter and launches a goroutine. diff --git a/metrics/registry.go b/metrics/registry.go index 1541a81fdd..298598a9aa 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -188,6 +188,18 @@ func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { values["5m.rate"] = t.Rate5() values["15m.rate"] = t.Rate15() values["mean.rate"] = t.RateMean() + case *ResettingTimer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] } data[name] = values }) @@ -333,13 +345,6 @@ func GetOrRegister(name string, i func() interface{}) interface{} { return DefaultRegistry.GetOrRegister(name, i) } -func getOrRegister[T any](name string, ctor func() T, r Registry) T { - if r == nil { - r = DefaultRegistry - } - return r.GetOrRegister(name, func() any { return ctor() }).(T) -} - // Register the given metric under the given name. Returns a ErrDuplicateMetric // if a metric by the given name is already registered. func Register(name string, i interface{}) error { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6af0796da9..1aad7a0028 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -14,6 +14,31 @@ func BenchmarkRegistry(b *testing.B) { } } +func BenchmarkRegistryGetOrRegister(b *testing.B) { + sample := func() Sample { return nil } + tests := []struct { + name string + ctor func() any + }{ + {name: "counter", ctor: func() any { return GetOrRegisterCounter("counter", DefaultRegistry) }}, + {name: "gauge", ctor: func() any { return GetOrRegisterGauge("gauge", DefaultRegistry) }}, + {name: "gaugefloat64", ctor: func() any { return GetOrRegisterGaugeFloat64("gaugefloat64", DefaultRegistry) }}, + {name: "histogram", ctor: func() any { return GetOrRegisterHistogram("histogram", DefaultRegistry, sample()) }}, + {name: "meter", ctor: func() any { return GetOrRegisterMeter("meter", DefaultRegistry) }}, + {name: "timer", ctor: func() any { return GetOrRegisterTimer("timer", DefaultRegistry) }}, + {name: "gaugeinfo", ctor: func() any { return GetOrRegisterGaugeInfo("gaugeinfo", DefaultRegistry) }}, + {name: "resettingtimer", ctor: func() any { return GetOrRegisterResettingTimer("resettingtimer", DefaultRegistry) }}, + {name: "runtimehistogramlazy", ctor: func() any { return GetOrRegisterHistogramLazy("runtimehistogramlazy", DefaultRegistry, sample) }}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + test.ctor() + } + }) + } +} + func BenchmarkRegistryGetOrRegisterParallel_8(b *testing.B) { benchmarkRegistryGetOrRegisterParallel(b, 8) } @@ -268,7 +293,7 @@ func TestPrefixedChildRegistryGet(t *testing.T) { } func TestChildPrefixedRegistryRegister(t *testing.T) { - r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") err := r.Register("foo", NewCounter()) c := NewCounter() Register("bar", c) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 8aa7dc1488..a3f46e52e0 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -8,7 +8,10 @@ import ( // GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a // new ResettingTimer. func GetOrRegisterResettingTimer(name string, r Registry) *ResettingTimer { - return getOrRegister(name, NewResettingTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewResettingTimer() }).(*ResettingTimer) } // NewRegisteredResettingTimer constructs and registers a new ResettingTimer. diff --git a/metrics/runtimehistogram.go b/metrics/runtimehistogram.go index efbed498af..e975a570a4 100644 --- a/metrics/runtimehistogram.go +++ b/metrics/runtimehistogram.go @@ -8,8 +8,10 @@ import ( ) func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runtimeHistogram { - constructor := func() Histogram { return newRuntimeHistogram(scale) } - return getOrRegister(name, constructor, r).(*runtimeHistogram) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return newRuntimeHistogram(scale) }).(*runtimeHistogram) } // runtimeHistogram wraps a runtime/metrics histogram. diff --git a/metrics/timer.go b/metrics/timer.go index 894bdfc327..8082b31947 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -10,7 +10,10 @@ import ( // Be sure to unregister the meter from the registry once it is of no use to // allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) *Timer { - return getOrRegister(name, NewTimer, r) + if r == nil { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() any { return NewTimer() }).(*Timer) } // NewCustomTimer constructs a new Timer from a Histogram and a Meter. diff --git a/node/config.go b/node/config.go index dc436876cc..255b0f0aa9 100644 --- a/node/config.go +++ b/node/config.go @@ -191,9 +191,7 @@ type Config struct { GraphQLVirtualHosts []string `toml:",omitempty"` // Logger is a custom logger to use with the p2p.Server. - Logger log.Logger `toml:",omitempty"` - - oldGethResourceWarning bool + Logger log.Logger `toml:"-,omitempty"` // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. AllowUnprotectedTxs bool `toml:",omitempty"` @@ -210,7 +208,29 @@ type Config struct { // EnablePersonal enables the deprecated personal namespace. EnablePersonal bool `toml:"-"` + // Configures database engine used by the node. DBEngine string `toml:",omitempty"` + + // Configures OpenTelemetry reporting. + OpenTelemetry OpenTelemetryConfig `toml:",omitempty"` + + oldGethResourceWarning bool +} + +// OpenTelemetryConfig has settings for +type OpenTelemetryConfig struct { + Enabled bool `toml:",omitempty"` + + Tags string `toml:",omitempty"` + InstanceID string `toml:",omitempty"` + + // Exporter endpoint. + Endpoint string `toml:",omitempty"` + AuthUser string `toml:",omitempty"` + AuthPassword string `toml:",omitempty"` + + // Percentage of sampled traces. + SampleRatio float64 `toml:",omitempty"` } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into diff --git a/node/rpcstack.go b/node/rpcstack.go index a1cc832f9f..a9ac88e4de 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -138,6 +138,9 @@ func (h *httpServer) start() error { // Initialize the server. h.server = &http.Server{Handler: h} + h.server.Protocols = new(http.Protocols) + h.server.Protocols.SetHTTP1(true) + h.server.Protocols.SetUnencryptedHTTP2(true) if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 54e58cccb2..bd75dac4eb 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -593,6 +593,38 @@ func TestHTTPWriteTimeout(t *testing.T) { }) } +func TestHTTP2H2C(t *testing.T) { + srv := createAndStartServer(t, &httpConfig{}, false, &wsConfig{}, nil) + defer srv.stop() + + // Create an HTTP/2 cleartext client. + transport := &http.Transport{} + transport.Protocols = new(http.Protocols) + transport.Protocols.SetUnencryptedHTTP2(true) + client := &http.Client{Transport: transport} + + body := strings.NewReader(`{"jsonrpc":"2.0","id":1,"method":"rpc_modules","params":[]}`) + resp, err := client.Post("http://"+srv.listenAddr(), "application/json", body) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + t.Fatalf("expected status 200, got %d", resp.StatusCode) + } + if resp.Proto != "HTTP/2.0" { + t.Fatalf("expected HTTP/2.0, got %s", resp.Proto) + } + result, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(result), "jsonrpc") { + t.Fatalf("unexpected response: %s", result) + } +} + func apis() []rpc.API { return []rpc.API{ { diff --git a/rlp/encode.go b/rlp/encode.go index ba8959ee6f..9d04e6324a 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -122,7 +122,7 @@ func EncodeToRawList[T any](val []T) (RawList[T], error) { bytes := make([]byte, contentSize+9) offset := 9 - headsize(uint64(contentSize)) buf.copyTo(bytes[offset:]) - return RawList[T]{enc: bytes}, nil + return RawList[T]{enc: bytes, length: len(val)}, nil } type listhead struct { diff --git a/rlp/iterator.go b/rlp/iterator.go index a67e6651f5..9e41cec947 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -25,20 +25,20 @@ type Iterator struct { } // NewListIterator creates an iterator for the (list) represented by data. -func NewListIterator(data RawValue) (*Iterator, error) { +func NewListIterator(data RawValue) (Iterator, error) { k, t, c, err := readKind(data) if err != nil { - return nil, err + return Iterator{}, err } if k != List { - return nil, ErrExpectedList + return Iterator{}, ErrExpectedList } - it := &Iterator{data: data[t : t+c], offset: int(t)} + it := newIterator(data[t:t+c], int(t)) return it, nil } -func newIterator(data []byte) *Iterator { - return &Iterator{data: data} +func newIterator(data []byte, initialOffset int) Iterator { + return Iterator{data: data, offset: initialOffset} } // Next forwards the iterator one step. @@ -64,6 +64,11 @@ func (it *Iterator) Next() bool { return true } +// Value returns the current value. +func (it *Iterator) Value() []byte { + return it.next +} + // Count returns the remaining number of items. // Note this is O(n) and the result may be incorrect if the list data is invalid. // The returned count is always an upper bound on the remaining items @@ -73,11 +78,6 @@ func (it *Iterator) Count() int { return count } -// Value returns the current value. -func (it *Iterator) Value() []byte { - return it.next -} - // Offset returns the offset of the current value into the list data. func (it *Iterator) Offset() int { return it.offset - len(it.next) diff --git a/rlp/iterator_test.go b/rlp/iterator_test.go index 8ea26dad1b..275d4371c7 100644 --- a/rlp/iterator_test.go +++ b/rlp/iterator_test.go @@ -17,6 +17,7 @@ package rlp import ( + "io" "testing" "github.com/ethereum/go-ethereum/common/hexutil" @@ -54,6 +55,9 @@ func TestIterator(t *testing.T) { if err != nil { t.Fatal(err) } + if c := txit.Count(); c != 2 { + t.Fatal("wrong Count:", c) + } var i = 0 for txit.Next() { if txit.err != nil { @@ -65,3 +69,65 @@ func TestIterator(t *testing.T) { t.Errorf("count wrong, expected %d got %d", i, exp) } } + +func TestIteratorErrors(t *testing.T) { + tests := []struct { + input []byte + wantCount int // expected Count before iterating + wantErr error + }{ + // Second item string header claims 3 bytes content, but only 2 remain. + {unhex("C4 01 83AABB"), 2, ErrValueTooLarge}, + // Second item truncated: B9 requires 2 size bytes, none available. + {unhex("C2 01 B9"), 2, io.ErrUnexpectedEOF}, + // 0x05 should be encoded directly, not as 81 05. + {unhex("C3 01 8105"), 2, ErrCanonSize}, + // Long-form string header B8 used for 1-byte content (< 56). + {unhex("C4 01 B801AA"), 2, ErrCanonSize}, + // Long-form list header F8 used for 1-byte content (< 56). + {unhex("C4 01 F80101"), 2, ErrCanonSize}, + } + for _, tt := range tests { + it, err := NewListIterator(tt.input) + if err != nil { + t.Fatal("NewListIterator error:", err) + } + if c := it.Count(); c != tt.wantCount { + t.Fatalf("%x: Count = %d, want %d", tt.input, c, tt.wantCount) + } + n := 0 + for it.Next() { + if it.Err() != nil { + break + } + n++ + } + if wantN := tt.wantCount - 1; n != wantN { + t.Fatalf("%x: got %d valid items, want %d", tt.input, n, wantN) + } + if it.Err() != tt.wantErr { + t.Fatalf("%x: got error %v, want %v", tt.input, it.Err(), tt.wantErr) + } + if it.Next() { + t.Fatalf("%x: Next returned true after error", tt.input) + } + } +} + +func FuzzIteratorCount(f *testing.F) { + examples := [][]byte{unhex("010203"), unhex("018142"), unhex("01830202")} + for _, e := range examples { + f.Add(e) + } + f.Fuzz(func(t *testing.T, in []byte) { + it := newIterator(in, 0) + count := it.Count() + i := 0 + for it.Next() { + i++ + } + if i != count { + t.Fatalf("%x: count %d not equal to %d iterations", in, count, i) + } + }) +} diff --git a/rlp/raw.go b/rlp/raw.go index 876a503d70..08ec667158 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -44,6 +44,9 @@ type RawList[T any] struct { // The implementation code mostly works with the Content method because it // returns something valid either way. enc []byte + + // length holds the number of items in the list. + length int } // Content returns the RLP-encoded data of the list. @@ -87,7 +90,14 @@ func (r *RawList[T]) DecodeRLP(s *Stream) error { if err := s.readFull(enc[9:]); err != nil { return err } - *r = RawList[T]{enc: enc} + n, err := CountValues(enc[9:]) + if err != nil { + if err == ErrValueTooLarge { + return ErrElemTooLarge + } + return err + } + *r = RawList[T]{enc: enc, length: n} return nil } @@ -105,8 +115,7 @@ func (r *RawList[T]) Items() ([]T, error) { // Len returns the number of items in the list. func (r *RawList[T]) Len() int { - len, _ := CountValues(r.Content()) - return len + return r.length } // Size returns the encoded size of the list. @@ -114,16 +123,11 @@ func (r *RawList[T]) Size() uint64 { return ListSize(uint64(len(r.Content()))) } -// Empty returns true if the list contains no items. -func (r *RawList[T]) Empty() bool { - return len(r.Content()) == 0 -} - // ContentIterator returns an iterator over the content of the list. // Note the offsets returned by iterator.Offset are relative to the // Content bytes of the list. -func (r *RawList[T]) ContentIterator() *Iterator { - return newIterator(r.Content()) +func (r *RawList[T]) ContentIterator() Iterator { + return newIterator(r.Content(), 0) } // Append adds an item to the end of the list. @@ -142,6 +146,25 @@ func (r *RawList[T]) Append(item T) error { end := prevEnd + eb.size() r.enc = slices.Grow(r.enc, eb.size())[:end] eb.copyTo(r.enc[prevEnd:end]) + r.length++ + return nil +} + +// AppendRaw adds an encoded item to the list. +// The given byte slice must contain exactly one RLP value. +func (r *RawList[T]) AppendRaw(b []byte) error { + _, tagsize, contentsize, err := readKind(b) + if err != nil { + return err + } + if tagsize+contentsize != uint64(len(b)) { + return fmt.Errorf("rlp: input has trailing bytes in AppendRaw") + } + if r.enc == nil { + r.enc = make([]byte, 9) + } + r.enc = append(r.enc, b...) + r.length++ return nil } @@ -262,7 +285,7 @@ func CountValues(b []byte) (int, error) { for ; len(b) > 0; i++ { _, tagsize, size, err := readKind(b) if err != nil { - return 0, err + return i + 1, err } b = b[tagsize+size:] } diff --git a/rlp/raw_test.go b/rlp/raw_test.go index 9a4c68050c..112c5d7897 100644 --- a/rlp/raw_test.go +++ b/rlp/raw_test.go @@ -68,9 +68,6 @@ func (test rawListTest[T]) run(t *testing.T) { // check iterator it := rl.ContentIterator() i := 0 - if count := it.Count(); count != test.length { - t.Fatalf("iterator has wrong Count %d, want %d", count, test.length) - } for it.Next() { var item T if err := DecodeBytes(it.Value(), &item); err != nil { @@ -154,9 +151,6 @@ func TestRawListEmpty(t *testing.T) { if !bytes.Equal(b, unhex("C0")) { t.Fatalf("empty RawList has wrong encoding %x", b) } - if !rl.Empty() { - t.Fatal("list should be Empty") - } if rl.Len() != 0 { t.Fatalf("empty list has Len %d", rl.Len()) } @@ -226,6 +220,58 @@ func TestRawListAppend(t *testing.T) { } } +func TestRawListAppendRaw(t *testing.T) { + var rl RawList[uint64] + + if err := rl.AppendRaw(unhex("01")); err != nil { + t.Fatal("AppendRaw(01) failed:", err) + } + if err := rl.AppendRaw(unhex("820102")); err != nil { + t.Fatal("AppendRaw(820102) failed:", err) + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after valid appends", rl.Len()) + } + + if err := rl.AppendRaw(nil); err == nil { + t.Fatal("AppendRaw(nil) should fail") + } + if err := rl.AppendRaw(unhex("0102")); err == nil { + t.Fatal("AppendRaw(0102) should fail due to trailing bytes") + } + if err := rl.AppendRaw(unhex("8201")); err == nil { + t.Fatal("AppendRaw(8201) should fail due to truncated value") + } + if rl.Len() != 2 { + t.Fatalf("wrong Len %d after invalid appends, want 2", rl.Len()) + } +} + +func TestRawListDecodeInvalid(t *testing.T) { + tests := []struct { + input string + err error + }{ + // Single item with non-canonical size (0x81 wrapping byte <= 0x7F). + {input: "C28142", err: ErrCanonSize}, + // Single item claiming more bytes than available in the list. + {input: "C484020202", err: ErrElemTooLarge}, + // Two items, second has non-canonical size. + {input: "C3018142", err: ErrCanonSize}, + // Two items, second claims more bytes than remain in the list. + {input: "C401830202", err: ErrElemTooLarge}, + // Item is a sub-list whose declared size exceeds available bytes. + {input: "C3C40102", err: ErrElemTooLarge}, + } + for _, test := range tests { + var rl RawList[RawValue] + err := DecodeBytes(unhex(test.input), &rl) + if !errors.Is(err, test.err) { + t.Errorf("input %s: error mismatch: got %v, want %v", test.input, err, test.err) + } + } +} + func TestCountValues(t *testing.T) { tests := []struct { input string // note: spaces in input are stripped by unhex @@ -242,9 +288,9 @@ func TestCountValues(t *testing.T) { {"820101 820202 8403030303 04", 4, nil}, // size errors - {"8142", 0, ErrCanonSize}, - {"01 01 8142", 0, ErrCanonSize}, - {"02 84020202", 0, ErrValueTooLarge}, + {"8142", 1, ErrCanonSize}, + {"01 01 8142", 3, ErrCanonSize}, + {"02 84020202", 2, ErrValueTooLarge}, { input: "A12000BF49F440A1CD0527E4D06E2765654C0F56452257516D793A9B8D604DCFDF2AB853F851808D10000000000000000000000000A056E81F171BCC55A6FF8345E692C0F86E5B48E01B996CADC001622FB5E363B421A0C5D2460186F7233C927E7DB2DCC703C0E500B653CA82273B7BFAD8045D85A470", diff --git a/rpc/handler.go b/rpc/handler.go index 4ac3a26df1..c0af162f13 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -524,7 +524,6 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage } // Start root span for the request. - var err error rpcInfo := telemetry.RPCInfo{ System: "jsonrpc", Service: service, @@ -535,24 +534,25 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage telemetry.BoolAttribute("rpc.batch", cp.isBatch), } ctx, spanEnd := telemetry.StartServerSpan(cp.ctx, h.tracer(), rpcInfo, attrib...) - defer spanEnd(err) + defer spanEnd(nil) // don't propagate errors to parent spans // Start tracing span before parsing arguments. _, _, pSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.parsePositionalArguments") - args, err := parsePositionalArguments(msg.Params, callb.argTypes) - pSpanEnd(err) - if err != nil { - return msg.errorResponse(&invalidParamsError{err.Error()}) + args, pErr := parsePositionalArguments(msg.Params, callb.argTypes) + pSpanEnd(&pErr) + if pErr != nil { + return msg.errorResponse(&invalidParamsError{pErr.Error()}) } start := time.Now() // Start tracing span before running the method. rctx, _, rSpanEnd := telemetry.StartSpanWithTracer(ctx, h.tracer(), "rpc.runMethod") answer := h.runMethod(rctx, msg, callb, args) + var rErr error if answer.Error != nil { - err = errors.New(answer.Error.Message) + rErr = errors.New(answer.Error.Message) } - rSpanEnd(err) + rSpanEnd(&rErr) // Collect the statistics for RPC calls if metrics is enabled. rpcRequestGauge.Inc(1) @@ -625,7 +625,7 @@ func (h *handler) runMethod(ctx context.Context, msg *jsonrpcMessage, callb *cal if response.Error != nil { err = errors.New(response.Error.Message) } - spanEnd(err) + spanEnd(&err) return response } diff --git a/rpc/tracing_test.go b/rpc/tracing_test.go index f32a647e6f..5a04c901fd 100644 --- a/rpc/tracing_test.go +++ b/rpc/tracing_test.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/propagation" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" @@ -141,6 +142,49 @@ func TestTracingHTTP(t *testing.T) { } } +// TestTracingErrorRecording verifies that errors are recorded on spans. +func TestTracingHTTPErrorRecording(t *testing.T) { + t.Parallel() + server, tracer, exporter := newTracingServer(t) + httpsrv := httptest.NewServer(server) + t.Cleanup(httpsrv.Close) + client, err := DialHTTP(httpsrv.URL) + if err != nil { + t.Fatalf("failed to dial: %v", err) + } + t.Cleanup(client.Close) + + // Call a method that returns an error. + var result any + err = client.Call(&result, "test_returnError") + if err == nil { + t.Fatal("expected error from test_returnError") + } + + // Flush and verify spans recorded the error. + if err := tracer.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + + // Only the runMethod span should have error status. + if len(spans) == 0 { + t.Fatal("no spans were emitted") + } + for _, span := range spans { + switch span.Name { + case "rpc.runMethod": + if span.Status.Code != codes.Error { + t.Errorf("expected %s span status Error, got %v", span.Name, span.Status.Code) + } + default: + if span.Status.Code == codes.Error { + t.Errorf("unexpected error status on span %s", span.Name) + } + } + } +} + // TestTracingBatchHTTP verifies that RPC spans are emitted for batched JSON-RPC calls over HTTP. func TestTracingBatchHTTP(t *testing.T) { t.Parallel() diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index 966d236c08..ecdd002331 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -236,7 +236,7 @@ func (t *BinaryTrie) GetAccount(addr common.Address) (*types.StateAccount, error // not be modified by the caller. If a node was not found in the database, a // trie.MissingNodeError is returned. func (t *BinaryTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { - return t.root.Get(GetBinaryTreeKey(addr, key), t.nodeResolver) + return t.root.Get(GetBinaryTreeKeyStorageSlot(addr, key), t.nodeResolver) } // UpdateAccount updates the account information for the given address. @@ -302,7 +302,7 @@ func (t *BinaryTrie) DeleteAccount(addr common.Address) error { // DeleteStorage removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. func (t *BinaryTrie) DeleteStorage(addr common.Address, key []byte) error { - k := GetBinaryTreeKey(addr, key) + k := GetBinaryTreeKeyStorageSlot(addr, key) var zero [HashSize]byte root, err := t.root.Insert(k, zero[:], t.nodeResolver, 0) if err != nil { diff --git a/trie/bintrie/trie_test.go b/trie/bintrie/trie_test.go index 050cc8d940..256fd218e2 100644 --- a/trie/bintrie/trie_test.go +++ b/trie/bintrie/trie_test.go @@ -22,7 +22,9 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" ) var ( @@ -197,6 +199,74 @@ func TestMerkleizeMultipleEntries(t *testing.T) { } } +// TestStorageRoundTrip verifies that GetStorage and DeleteStorage use the same +// key mapping as UpdateStorage (GetBinaryTreeKeyStorageSlot). This is a regression +// test: previously GetStorage and DeleteStorage used GetBinaryTreeKey directly, +// which produced different tree keys and broke the read/delete path. +func TestStorageRoundTrip(t *testing.T) { + tracer := trie.NewPrevalueTracer() + tr := &BinaryTrie{ + root: NewBinaryNode(), + tracer: tracer, + } + addr := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + + // Create an account first so the root becomes an InternalNode, + // which is the realistic state when storage operations happen. + acc := &types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + CodeHash: common.HexToHash("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), + } + if err := tr.UpdateAccount(addr, acc, 0); err != nil { + t.Fatalf("UpdateAccount error: %v", err) + } + + // Test main storage slots (key[31] >= 64 or key[:31] != 0). + // These produce a different stem than the account data, so after + // UpdateAccount + UpdateStorage the root is an InternalNode. + // Note: header slots (key[31] < 64, key[:31] == 0) share the same + // stem as account data and are covered by GetAccount/UpdateAccount path. + slots := []common.Hash{ + common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), // main storage (slot 255) + common.HexToHash("0100000000000000000000000000000000000000000000000000000000000001"), // main storage (non-zero prefix) + } + val := common.TrimLeftZeroes(common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef").Bytes()) + + for _, slot := range slots { + // Write + if err := tr.UpdateStorage(addr, slot[:], val); err != nil { + t.Fatalf("UpdateStorage(%x) error: %v", slot, err) + } + // Read back + got, err := tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) error: %v", slot, err) + } + if len(got) == 0 { + t.Fatalf("GetStorage(%x) returned empty, expected value", slot) + } + // Verify value (right-justified in 32 bytes) + var expected [HashSize]byte + copy(expected[HashSize-len(val):], val) + if !bytes.Equal(got, expected[:]) { + t.Fatalf("GetStorage(%x) = %x, want %x", slot, got, expected) + } + // Delete + if err := tr.DeleteStorage(addr, slot[:]); err != nil { + t.Fatalf("DeleteStorage(%x) error: %v", slot, err) + } + // Verify deleted (should read as zero, not the old value) + got, err = tr.GetStorage(addr, slot[:]) + if err != nil { + t.Fatalf("GetStorage(%x) after delete error: %v", slot, err) + } + if len(got) > 0 && !bytes.Equal(got, zero[:]) { + t.Fatalf("GetStorage(%x) after delete = %x, expected zero", slot, got) + } + } +} + func TestBinaryTrieWitness(t *testing.T) { tracer := trie.NewPrevalueTracer() diff --git a/trie/node.go b/trie/node.go index 3f14f07d63..7022116048 100644 --- a/trie/node.go +++ b/trie/node.go @@ -161,11 +161,14 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) { if err != nil { return nil, fmt.Errorf("decode error: %v", err) } - switch c, _ := rlp.CountValues(elems); c { - case 2: + c, err := rlp.CountValues(elems) + switch { + case err != nil: + return nil, fmt.Errorf("invalid node list: %v", err) + case c == 2: n, err := decodeShort(hash, elems) return n, wrapError(err, "short") - case 17: + case c == 17: n, err := decodeFull(hash, elems) return n, wrapError(err, "full") default: @@ -225,7 +228,7 @@ func decodeRef(buf []byte) (node, []byte, error) { case kind == rlp.List: // 'embedded' node reference. The encoding must be smaller // than a hash in order to be valid. - if size := len(buf) - len(rest); size > hashLen { + if size := len(buf) - len(rest); size >= hashLen { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index f7c0ba1398..5255602a4e 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -45,7 +45,7 @@ type layer interface { // Note: // - the returned node is not a copy, please don't modify it. // - no error will be returned if the requested node is not found in database. - node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) + node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) // account directly retrieves the account RLP associated with a particular // hash in the slim data format. An error will be returned if the read diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index ae523c979c..8ca3a39cf3 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -79,7 +79,7 @@ func (dl *diffLayer) parentLayer() layer { // node implements the layer interface, retrieving the trie node blob with the // provided node information. No error will be returned if the node is not found. -func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { // Hold the lock, ensure the parent won't be changed during the // state accessing. dl.lock.RLock() @@ -91,7 +91,7 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeHitDepthHist.Update(int64(depth)) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) - return n.Blob, n.Hash, &nodeLoc{loc: locDiffLayer, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDiffLayer, depth: depth}, nil } // Trie node unknown to this layer, resolve from parent return dl.parent.node(owner, path, depth+1) diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 911959dfa9..5bad19b4f5 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -112,12 +112,12 @@ func (dl *diskLayer) markStale() { // node implements the layer interface, retrieving the trie node with the // provided node info. No error will be returned if the node is not found. -func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) { +func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, nodeLoc, error) { dl.lock.RLock() defer dl.lock.RUnlock() if dl.stale { - return nil, common.Hash{}, nil, errSnapshotStale + return nil, common.Hash{}, nodeLoc{}, errSnapshotStale } // Try to retrieve the trie node from the not-yet-written node buffer first // (both the live one and the frozen one). Note the buffer is lock free since @@ -129,7 +129,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dirtyNodeHitMeter.Mark(1) dirtyNodeReadMeter.Mark(int64(len(n.Blob))) dirtyNodeHitDepthHist.Update(int64(depth)) - return n.Blob, n.Hash, &nodeLoc{loc: locDirtyCache, depth: depth}, nil + return n.Blob, n.Hash, nodeLoc{loc: locDirtyCache, depth: depth}, nil } } } @@ -141,7 +141,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co if blob := dl.nodes.Get(nil, key); len(blob) > 0 { cleanNodeHitMeter.Mark(1) cleanNodeReadMeter.Mark(int64(len(blob))) - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locCleanCache, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locCleanCache, depth: depth}, nil } cleanNodeMissMeter.Mark(1) } @@ -161,7 +161,7 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co dl.nodes.Set(key, blob) cleanNodeWriteMeter.Mark(int64(len(blob))) } - return blob, crypto.Keccak256Hash(blob), &nodeLoc{loc: locDiskLayer, depth: depth}, nil + return blob, crypto.Keccak256Hash(blob), nodeLoc{loc: locDiskLayer, depth: depth}, nil } // account directly retrieves the account RLP associated with a particular diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index f55e015ee6..aaa64e902c 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -47,7 +47,7 @@ type nodeLoc struct { } // string returns the string representation of node location. -func (loc *nodeLoc) string() string { +func (loc nodeLoc) string() string { return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth) }