From d0170302b1533573344bf113c547ec1590b87db6 Mon Sep 17 00:00:00 2001 From: "lightclient@protonmail.com" Date: Fri, 3 Feb 2023 13:02:26 -0700 Subject: [PATCH] all: implement era format, add history importer/export --- cmd/era/main.go | 331 +++++++++++++++++ cmd/geth/chaincmd.go | 116 ++++++ cmd/geth/main.go | 2 + cmd/utils/cmd.go | 158 ++++++++ cmd/utils/history_test.go | 171 +++++++++ core/blockchain_reader.go | 5 + go.mod | 3 + go.sum | 11 +- internal/era/accumulator.go | 87 +++++ internal/era/e2store/e2store.go | 146 ++++++++ internal/era/e2store/e2store_test.go | 158 ++++++++ internal/era/era.go | 528 +++++++++++++++++++++++++++ internal/era/era_test.go | 132 +++++++ 13 files changed, 1845 insertions(+), 3 deletions(-) create mode 100644 cmd/era/main.go create mode 100644 cmd/utils/history_test.go create mode 100644 internal/era/accumulator.go create mode 100644 internal/era/e2store/e2store.go create mode 100644 internal/era/e2store/e2store_test.go create mode 100644 internal/era/era.go create mode 100644 internal/era/era_test.go diff --git a/cmd/era/main.go b/cmd/era/main.go new file mode 100644 index 0000000000..6ab1020f2d --- /dev/null +++ b/cmd/era/main.go @@ -0,0 +1,331 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "io" + "math/big" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/urfave/cli/v2" +) + +var app = flags.NewApp("go-ethereum era tool") + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Usage: "directory storing all relevant era1 files", + Value: "eras", + } + networkFlag = &cli.StringFlag{ + Name: "network", + Usage: "network name associated with era1 files", + Value: "mainnet", + } + eraSizeFlag = &cli.IntFlag{ + Name: "size", + Usage: "number of blocks per era", + Value: era.MaxEra1Size, + } + txsFlag = &cli.BoolFlag{ + Name: "txs", + Usage: "print full transaction values", + } +) + +var ( + blockCommand = &cli.Command{ + Name: "block", + Usage: "get block data", + ArgsUsage: "", + Action: block, + Flags: []cli.Flag{ + txsFlag, + }, + } + infoCommand = &cli.Command{ + Name: "info", + ArgsUsage: "", + Usage: "get epoch information", + Action: info, + } + verifyCommand = &cli.Command{ + Name: "verify", + ArgsUsage: "", + Usage: "verifies each era against expected accumulator root", + Action: verify, + } +) + +func init() { + app.Commands = []*cli.Command{ + blockCommand, + infoCommand, + verifyCommand, + } + app.Flags = []cli.Flag{ + dirFlag, + networkFlag, + eraSizeFlag, + } +} + +func main() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// block prints the specified block from an era1 store. +func block(ctx *cli.Context) error { + num, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid block number: %w", err) + } + + f, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + r, err := era.NewReader(f) + if err != nil { + return fmt.Errorf("error making era reader: %w", err) + } + + // Read block with number. + block, err := r.ReadBlock(num) + if err != nil { + return fmt.Errorf("error reading era: %w", err) + } + + // Convert block to JSON and print. + val, err := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig) + if err != nil { + return fmt.Errorf("error marshaling json: %w", err) + } + b, err := json.MarshalIndent(val, "", " ") + if err != nil { + return fmt.Errorf("error marshaling json: %w", err) + } + fmt.Println(string(b)) + return nil +} + +// info prints some high-level information about the era1 file. +func info(ctx *cli.Context) error { + epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64) + if err != nil { + return fmt.Errorf("invalid epoch number: %w", err) + } + f, err := open(ctx, epoch) + if err != nil { + return err + } + r, err := era.NewReader(f) + if err != nil { + return fmt.Errorf("error creating era reader: %w", err) + } + acc, err := r.Accumulator() + if err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + td, err := r.InitialTD() + if err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + info := struct { + Accumulator common.Hash `json:"accumulator"` + TotalDifficulty *big.Int `json:"totalDifficulty"` + StartBlock uint64 `json:"startBlock"` + Count uint64 `json:"count"` + }{ + acc, td, r.Start(), r.Count(), + } + b, _ := json.MarshalIndent(info, "", " ") + fmt.Println(string(b)) + return nil +} + +// open opens an era1 file at a certain epoch. +func open(ctx *cli.Context, epoch uint64) (*os.File, error) { + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + ) + entries, err := era.ReadDir(dir, network) + if err != nil { + return nil, fmt.Errorf("error reading era dir: %w", err) + } + if epoch >= uint64(len(entries)) { + return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) + } + return os.Open(path.Join(dir, entries[epoch])) +} + +// verify checks each era1 file in a directory to ensure it is well-formed and +// that the accumulator matches the expected value. +func verify(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + return fmt.Errorf("missing accumulators file") + } + + roots, err := readHashes(ctx.Args().First()) + if err != nil { + return fmt.Errorf("unable to read expected roots file: %w", err) + } + + var ( + dir = ctx.String(dirFlag.Name) + network = ctx.String(networkFlag.Name) + start = time.Now() + reported = time.Now() + ) + + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + + if len(entries) != len(roots) { + return fmt.Errorf("number of era1 files should match the number of accumulator hashes") + } + + // Verify each epoch matches the expected root. + for i, want := range roots { + name := entries[i] + f, err := os.Open(path.Join(dir, name)) + if err != nil { + return fmt.Errorf("error opening era1 file %s: %w", name, err) + } + defer f.Close() + + r, err := era.NewReader(f) + if err != nil { + return fmt.Errorf("unable to make era reader: %w", err) + } + + // Read accumulator and check against expected. + if got, err := r.Accumulator(); err != nil { + return fmt.Errorf("error retrieving accumulator for %s: %w", name, err) + } else if got != want { + return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want) + } + + // Recompute accumulator. + if err := checkAccumulator(r); err != nil { + return fmt.Errorf("error verify era1 file %s: %w", name, err) + } + + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + } + + return nil +} + +// checkAccumulator verifies the accumulator matches the data in the Era. +func checkAccumulator(r *era.Reader) error { + var ( + err error + start = r.Start() + want common.Hash + td *big.Int + tds = make([]*big.Int, 0) + hashes = make([]common.Hash, 0) + ) + if want, err = r.Accumulator(); err != nil { + return fmt.Errorf("error reading accumulator: %w", err) + } + if td, err = r.InitialTD(); err != nil { + return fmt.Errorf("error reading total difficulty: %w", err) + } + // Starting at epoch 0, iterate through all available era1 files and + // check the following: + // * the block index is constructed correctly + // * the starting total difficulty value is correct + // * the accumulator is correct by recomputing it locally, + // which verifies the blocks are all correct (via hash) + // * the receipts root matches the value in the block + for j := 0; ; j++ { + // read() walks the block index, so we're able to + // implicitly verify it. + block, receipts, err := r.Read() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("error reading block %d: %w", start+uint64(j), err) + } + tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)) + if tr != block.TxHash() { + return fmt.Errorf("tx root in block %d mismatch: want %s, got %s", block.NumberU64(), block.TxHash(), tr) + } + // Calculate receipt root from receipt list and check + // value against block. + rr := types.DeriveSha(receipts, trie.NewStackTrie(nil)) + if rr != block.ReceiptHash() { + return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr) + } + hashes = append(hashes, block.Hash()) + td.Add(td, block.Difficulty()) + tds = append(tds, new(big.Int).Set(td)) + } + got, err := era.ComputeAccumulator(hashes, tds) + if err != nil { + return fmt.Errorf("error computing accumulator: %w", err) + } + if got != want { + return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want) + } + return nil +} + +// readHashes reads a file of newline-delimited hashes. +func readHashes(f string) ([]common.Hash, error) { + b, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("unable to open accumulators file") + } + s := strings.Split(string(b), "\n") + // Remove empty last element, if present. + if s[len(s)-1] == "" { + s = s[:len(s)-1] + } + // Convert to hashes. + r := make([]common.Hash, len(s)) + for i := range s { + r[i] = common.HexToHash(s[i]) + } + return r, nil +} diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 3b4f516af7..52bd1bb9be 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,10 +35,12 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" ) @@ -122,6 +124,30 @@ Optional second and third arguments control the first and last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, + } + importHistoryCommand = &cli.Command{ + Action: importHistory, + Name: "import-history", + Usage: "Import an Era archive", + ArgsUsage: "", + Flags: flags.Merge([]cli.Flag{ + utils.TxLookupLimitFlag, + }, + utils.DatabaseFlags, + utils.NetworkFlags, + ), + Description: "", + } + exportHistoryCommand = &cli.Command{ + Action: exportHistory, + Name: "export-history", + Usage: "Export blockchain history to Era archives", + ArgsUsage: " ", + Flags: flags.Merge(utils.DatabaseFlags), + Description: ` +The export-history command will export blocks and their corresponding receipts +into Era archives. Eras are typically packaged in steps of 8192 blocks. +`, } importPreimagesCommand = &cli.Command{ Action: importPreimages, @@ -364,7 +390,97 @@ func exportChain(ctx *cli.Context) error { } err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last)) } + if err != nil { + utils.Fatalf("Export error: %v\n", err) + } + fmt.Printf("Export done in %v\n", time.Since(start)) + return nil +} +func importHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 1 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, db := utils.MakeChain(ctx, stack, false) + defer db.Close() + + var ( + start = time.Now() + dir = ctx.Args().Get(0) + network string + ) + + // Determine network. + if utils.IsNetworkPreset(ctx) { + switch { + case ctx.Bool(utils.MainnetFlag.Name): + network = "mainnet" + case ctx.Bool(utils.SepoliaFlag.Name): + network = "sepolia" + case ctx.Bool(utils.GoerliFlag.Name): + network = "goerli" + } + } else { + // No network flag set, try to determine network based on files + // present in directory. + var networks []string + for _, n := range params.NetworkNames { + entries, err := era.ReadDir(dir, n) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + if len(entries) > 0 { + networks = append(networks, n) + } + } + if len(networks) == 0 { + return fmt.Errorf("no era1 files found in %s", dir) + } + if len(networks) > 1 { + return fmt.Errorf("multiple networks found, use a network flag to specify desired network") + } + network = networks[0] + } + + if err := utils.ImportHistory(chain, db, dir, network); err != nil { + return err + } + fmt.Printf("Import done in %v\n", time.Since(start)) + return nil +} + +// exportHistory exports chain history in Era archives at a specified +// directory. +func exportHistory(ctx *cli.Context) error { + if ctx.Args().Len() != 3 { + utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chain, _ := utils.MakeChain(ctx, stack, true) + start := time.Now() + + var ( + dir = ctx.Args().Get(0) + first, ferr = strconv.ParseInt(ctx.Args().Get(1), 10, 64) + last, lerr = strconv.ParseInt(ctx.Args().Get(2), 10, 64) + ) + if ferr != nil || lerr != nil { + utils.Fatalf("Export error in parsing parameters: block number not an integer\n") + } + if first < 0 || last < 0 { + utils.Fatalf("Export error: block number must be greater than 0\n") + } + if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { + utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) + } + err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size)) if err != nil { utils.Fatalf("Export error: %v\n", err) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0fd0cc2099..2f7d37fdd7 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -208,6 +208,8 @@ func init() { initCommand, importCommand, exportCommand, + importHistoryCommand, + exportHistoryCommand, importPreimagesCommand, removedbCommand, dumpCommand, diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 8b571be1ef..3464b69a41 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -19,12 +19,15 @@ package utils import ( "bufio" + "bytes" "compress/gzip" + "crypto/sha256" "errors" "fmt" "io" "os" "os/signal" + "path" "runtime" "strings" "syscall" @@ -39,8 +42,10 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" ) @@ -228,6 +233,87 @@ func ImportChain(chain *core.BlockChain, fn string) error { return nil } +func readList(filename string) ([]string, error) { + b, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + return strings.Split(string(b), "\n"), nil +} + +// ImportHistory imports Era1 files containing historical block information, +// starting from genesis. +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { + if chain.CurrentSnapBlock().Number.BitLen() != 0 { + return fmt.Errorf("history import only supported when starting from genesis") + } + entries, err := era.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("error reading %s: %w", dir, err) + } + checksums, err := readList(path.Join(dir, "checksums.txt")) + if err != nil { + return fmt.Errorf("unable to read checksums.txt: %w", err) + } + if len(checksums) != len(entries) { + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + } + var ( + start = time.Now() + reported = time.Now() + imported = 0 + forker = core.NewForkChoice(chain, nil) + ) + for i, filename := range entries { + // Read entire Era1 to memory. Max historical Era1 is around + // 600MB. This is a lot to load at once, but it speeds up the + // import substantially. + f, err := os.ReadFile(path.Join(dir, filename)) + if err != nil { + return fmt.Errorf("unable to open era: %w", err) + } + + if have, want := common.Hash(sha256.Sum256(f)).Hex(), checksums[i]; have != want { + return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + } + + // Import all block data from Era1. + r, err := era.NewReader(bytes.NewReader(f)) + if err != nil { + return fmt.Errorf("error making era reader: %w", err) + } + for j := 0; ; j += 1 { + n := i*era.MaxEra1Size + j + block, receipts, err := r.Read() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("error reading block %d: %w", n, err) + } else if block.Number().BitLen() == 0 { + continue // skip genesis + } + if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil { + return fmt.Errorf("error inserting header %d: %w", n, err) + } else if status != core.CanonStatTy { + return fmt.Errorf("error inserting header %d, not canon: %v", n, status) + } + if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil { + return fmt.Errorf("error inserting body %d: %w", n, err) + } + imported += 1 + + // Give the user some feedback that something is happening. + if time.Since(reported) >= 8*time.Second { + log.Info("Importing Era files", "head", n, "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + imported = 0 + reported = time.Now() + } + } + } + + return nil +} + func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { head := chain.CurrentBlock() for i, block := range blocks { @@ -297,6 +383,78 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las return nil } +// ExportHistory exports blockchain history into the specified directory, +// following the Era format. +func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error { + log.Info("Exporting blockchain history", "dir", dir) + if head := bc.CurrentBlock().Number.Uint64(); head < last { + log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) + last = head + } + network := "unknown" + if name, ok := params.NetworkNames[bc.Config().ChainID.String()]; ok { + network = name + } + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating output directory: %w", err) + } + var ( + start = time.Now() + reported = time.Now() + checksums []string + ) + for i := first; i <= last; i += step { + var ( + buf = bytes.NewBuffer(nil) + w = era.NewBuilder(buf) + ) + for j := uint64(0); j < step && j <= last-i; j++ { + var ( + n = i + j + block = bc.GetBlockByNumber(n) + ) + if block == nil { + return fmt.Errorf("export failed on #%d: not found", n) + } + receipts := bc.GetReceiptsByHash(block.Hash()) + if receipts == nil { + return fmt.Errorf("export failed on #%d: receipts not found", n) + } + td := bc.GetTd(block.Hash(), block.NumberU64()) + if td == nil { + return fmt.Errorf("export failed on #%d: total difficulty not found", n) + } + if err := w.Add(block, receipts, td); err != nil { + return err + } + } + root, err := w.Finalize() + if err != nil { + return fmt.Errorf("export failed to finalize %d: %w", step/i, err) + } + + // Compute checksum of entire Era1. + checksums = append(checksums, common.Hash(sha256.Sum256(buf.Bytes())).Hex()) + + // Write Era1 to disk. + filename := path.Join(dir, era.Filename(network, int(i/step), root)) + if err := os.WriteFile(filename, buf.Bytes(), os.ModePerm); err != nil { + return err + } + + if time.Since(reported) >= 8*time.Second { + log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start))) + reported = time.Now() + } + } + + os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm) + + log.Info("Exported blockchain to", "dir", dir) + + return nil +} + // ImportPreimages imports a batch of exported hash preimages into the database. // It's a part of the deprecated functionality, should be removed in the future. func ImportPreimages(db ethdb.Database, fn string) error { diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go new file mode 100644 index 0000000000..e24a68bbd6 --- /dev/null +++ b/cmd/utils/history_test.go @@ -0,0 +1,171 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import ( + "bytes" + "crypto/sha256" + "io" + "math/big" + "os" + "path" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + count uint64 = 128 + step uint64 = 16 +) + +func TestHistoryImportAndExport(t *testing.T) { + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}}, + } + signer = types.LatestSigner(genesis.Config) + ) + + // Generate chain. + db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) { + if i == 0 { + return + } + tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(i - 1), + GasTipCap: common.Big0, + GasFeeCap: g.PrevBlock(0).BaseFee(), + Gas: 50000, + To: &common.Address{0xaa}, + Value: big.NewInt(int64(i)), + Data: nil, + AccessList: nil, + }) + if err != nil { + t.Fatalf("error creating tx: %v", err) + } + g.AddTx(tx) + }) + + // Initialize BlockChain. + chain, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("error insterting chain: %v", err) + } + + // Make temp directory for era files. + dir, err := os.MkdirTemp("", "history-export-test") + if err != nil { + t.Fatalf("error creating temp test directory: %v", err) + } + defer os.RemoveAll(dir) + + // Export history to temp directory. + if err := ExportHistory(chain, dir, 0, count, step); err != nil { + t.Fatalf("error exporting history: %v", err) + } + + // Read checksums. + b, err := os.ReadFile(path.Join(dir, "checksums.txt")) + if err != nil { + t.Fatalf("failed to read checksums: %v", err) + } + checksums := strings.Split(string(b), "\n") + + // Verify each Era. + entries, _ := era.ReadDir(dir, "mainnet") + for i, filename := range entries { + f, err := os.ReadFile(path.Join(dir, filename)) + if err != nil { + t.Fatalf("error opening era file: %v", err) + } + if want, got := common.HexToHash(checksums[i]), common.Hash(sha256.Sum256(f)); want != got { + t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) + } + r, err := era.NewReader(bytes.NewReader(f)) + if err != nil { + t.Fatalf("error making era reader: %v", err) + } + for j := 0; ; j += 1 { + block, receipts, err := r.Read() + if err == io.EOF { + break + } else if err != nil { + t.Fatalf("error reading era file %d: %v", i, err) + } + var ( + n = i*int(step) + j + want = chain.GetBlockByNumber(uint64(n)) + ) + if want, got := uint64(n), block.NumberU64(); want != got { + t.Fatalf("blocks out of order: want %d, got %d", want, got) + } + if want.Hash() != block.Hash() { + t.Fatalf("block hash mismatch %d: want %s, got %s", i+j, want.Hash().Hex(), block.Hash().Hex()) + } + if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { + t.Fatalf("tx hash %d mismatch: want %s, got %s", i+j, want.TxHash(), got) + } + if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { + t.Fatalf("uncle hash %d mismatch: want %s, got %s", i+j, want.UncleHash(), got) + } + if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { + t.Fatalf("receipt root %d mismatch: want %s, got %s", i+j, want.ReceiptHash(), got) + } + } + } + + // Now import Era. + freezer := t.TempDir() + db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false) + if err != nil { + panic(err) + } + t.Cleanup(func() { + db2.Close() + }) + + genesis.MustCommit(db2) + imported, err := core.NewBlockChain(db2, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("unable to initialize chain: %v", err) + } + if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil { + t.Fatalf("failed to import chain: %v", err) + } + if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { + t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash()) + } +} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 6fb09abacc..706844171d 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -410,6 +410,11 @@ func (bc *BlockChain) TrieDB() *trie.Database { return bc.triedb } +// HeaderChain returns the underlying header chain. +func (bc *BlockChain) HeaderChain() *HeaderChain { + return bc.hc +} + // SubscribeRemovedLogsEvent registers a subscription of RemovedLogsEvent. func (bc *BlockChain) SubscribeRemovedLogsEvent(ch chan<- RemovedLogsEvent) event.Subscription { return bc.scope.Track(bc.rmLogsFeed.Subscribe(ch)) diff --git a/go.mod b/go.mod index 79bdc2551a..acae682e19 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 github.com/ethereum/c-kzg-4844 v0.4.0 github.com/fatih/color v1.13.0 + github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/fsnotify/fsnotify v1.6.0 @@ -114,10 +115,12 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.15.15 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect diff --git a/go.sum b/go.sum index b692629b6b..57a357f16d 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,8 @@ github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= +github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= @@ -221,7 +223,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -400,6 +401,9 @@ github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -447,6 +451,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -524,6 +530,7 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/prysmaticlabs/gohashtree v0.0.1-alpha.0.20220714111606-acbb2962fb48 h1:cSo6/vk8YpvkLbk9v3FO97cakNmUoxwi2KMP8hd5WIw= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -777,8 +784,6 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= diff --git a/internal/era/accumulator.go b/internal/era/accumulator.go new file mode 100644 index 0000000000..78fa451930 --- /dev/null +++ b/internal/era/accumulator.go @@ -0,0 +1,87 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + ssz "github.com/ferranbt/fastssz" +) + +// ComputeAccumulator calculates the SSZ hash tree root of the Era1 +// accumulator of header records. +func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) { + if len(hashes) != len(tds) { + return common.Hash{}, fmt.Errorf("must have equal number hashes as td values") + } + hh := ssz.NewHasher() + for i := range hashes { + rec := headerRecord{hashes[i], tds[i]} + root, err := rec.HashTreeRoot() + if err != nil { + return common.Hash{}, err + } + hh.Append(root[:]) + } + hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size)) + return hh.HashRoot() +} + +// headerRecord is an individual record for a historical header. +// +// See https://github.com/ethereum/portal-network-specs/blob/master/history-network.md#the-header-accumulator +// for more information. +type headerRecord struct { + Hash common.Hash + TotalDifficulty *big.Int +} + +// GetTree completes the ssz.HashRoot interface, but is unused. +func (h *headerRecord) GetTree() (*ssz.Node, error) { + return nil, nil +} + +// HashTreeRoot ssz hashes the headerRecord object. +func (h *headerRecord) HashTreeRoot() ([32]byte, error) { + return ssz.HashWithDefaultHasher(h) +} + +// HashTreeRootWith ssz hashes the headerRecord object with a hasher. +func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) { + hh.PutBytes(h.Hash[:]) + td := bigToBytes32(h.TotalDifficulty) + hh.PutBytes(td[:]) + hh.Merkleize(0) + return +} + +// bigToBytes32 converts a big.Int into a little-endian 32-byte array. +func bigToBytes32(n *big.Int) (b [32]byte) { + n.FillBytes(b[:]) + reverseOrder(b[:]) + return +} + +// reverseOrder reverses the byte order of a slice. +func reverseOrder(b []byte) []byte { + for i := 0; i < 16; i++ { + b[i], b[32-i-1] = b[32-i-1], b[i] + } + return b +} diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go new file mode 100644 index 0000000000..4e43f6c58a --- /dev/null +++ b/internal/era/e2store/e2store.go @@ -0,0 +1,146 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "fmt" + "io" +) + +// e2store header size. +var headerSize = 8 + +// Entry is a variable-length-data record in an e2store. +type Entry struct { + Type uint16 + Value []byte +} + +// Writer writes entries using e2store encoding. +// For more information on this format, see: +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Writer struct { + w io.Writer +} + +// NewWriter returns a new Writer that writes to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w} +} + +// Write writes a single e2store entry to w. +// An entry is encoded in a type-length-value format. The first 8 bytes of the +// record store the type (2 bytes), the length (4 bytes), and some reserved +// data (2 bytes). The remaining bytes store b. +func (w *Writer) Write(typ uint16, b []byte) (int, error) { + buf := make([]byte, headerSize+len(b)) + + // type + buf[0] = byte(typ) + buf[1] = byte(typ >> 8) + + // length + l := len(b) + buf[2] = byte(l) + buf[3] = byte(l >> 8) + buf[4] = byte(l >> 16) + buf[5] = byte(l >> 24) + + // value + copy(buf[8:], b) + + return w.w.Write(buf) +} + +// A Reader reads entries from an e2store-encoded file. +// For more information on this format, see +// https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md +type Reader struct { + r io.Reader +} + +// NewReader returns a new Reader that reads from r. +func NewReader(r io.Reader) *Reader { + return &Reader{r} +} + +// Read reads one Entry from r. +// If the entry is malformed, it returns io.UnexpectedEOF. If there are no +// entries left to be read, Read returns io.EOF. +func (r *Reader) Read() (*Entry, error) { + b := make([]byte, headerSize) + if _, err := io.ReadFull(r.r, b); err != nil { + return nil, err + } + + typ := uint16(b[0]) + typ += uint16(b[1]) << 8 + + length := uint64(b[2]) + length += uint64(b[3]) << 8 + length += uint64(b[4]) << 16 + length += uint64(b[5]) << 24 + + // Check reserved bytes of header. + if b[6] != 0 || b[7] != 0 { + return nil, fmt.Errorf("reserved bytes are non-zero") + } + + val := make([]byte, length) + if _, err := io.ReadFull(r.r, val); err != nil { + // An entry with a non-zero length should not return EOF when + // reading the value. + if err == io.EOF { + return nil, io.ErrUnexpectedEOF + } + return nil, err + } + + return &Entry{ + Type: typ, + Value: val, + }, nil +} + +// Find returns the first entry with the matching type. +func (r *Reader) Find(typ uint16) (*Entry, error) { + for { + entry, err := r.Read() + if err == io.EOF { + return nil, io.EOF + } else if err != nil { + return nil, err + } + if entry.Type == typ { + return entry, nil + } + } +} + +// FindAll returns all entries with the matching type. +func (r *Reader) FindAll(typ uint16) ([]*Entry, error) { + all := make([]*Entry, 0) + for { + entry, err := r.Find(typ) + if err == io.EOF { + return all, io.EOF + } else if err != nil { + return all, err + } + all = append(all, entry) + } +} diff --git a/internal/era/e2store/e2store_test.go b/internal/era/e2store/e2store_test.go new file mode 100644 index 0000000000..c27a843139 --- /dev/null +++ b/internal/era/e2store/e2store_test.go @@ -0,0 +1,158 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package e2store + +import ( + "bytes" + "fmt" + "io" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestEncode(t *testing.T) { + for i, tt := range []struct { + entries []Entry + want string + }{ + { + entries: []Entry{{0xffff, nil}}, + want: "ffff000000000000", + }, + { + entries: []Entry{{42, common.Hex2Bytes("beef")}}, + want: "2a00020000000000beef", + }, + { + entries: []Entry{ + {42, common.Hex2Bytes("beef")}, + {9, common.Hex2Bytes("abcdabcd")}, + }, + want: "2a00020000000000beef0900040000000000abcdabcd", + }, + } { + var ( + b = NewWriteSeeker() + w = NewWriter(b) + ) + for _, e := range tt.entries { + if _, err := w.Write(e.Type, e.Value); err != nil { + t.Fatalf("test %d: encoding error: %v", i, err) + } + } + if want, got := common.Hex2Bytes(tt.want), b.Bytes(); !bytes.Equal(want, got) { + t.Fatalf("test %d: encoding mismatch (want %s, got %s", i, common.Bytes2Hex(want), common.Bytes2Hex(got)) + } + r := NewReader(io.NopCloser(bytes.NewBuffer(b.Bytes()))) + for _, want := range tt.entries { + if got, err := r.Read(); err != nil { + t.Fatalf("test %d: decoding error: %v", i, err) + } else if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("test %d: decoded entry does not match (want %v, got %v)", i, want, got) + } + } + } +} + +func TestDecode(t *testing.T) { + for i, tt := range []struct { + have string + want []Entry + err error + }{ + { // basic valid decoding + have: "ffff000000000000", + want: []Entry{{0xffff, nil}}, + }, + { // basic valid decoding + have: "ffff000000000001", + err: fmt.Errorf("reserved bytes are non-zero"), + }, + { // no more entries to read, returns EOF + have: "", + err: io.EOF, + }, + { // malformed type + have: "bad", + err: io.ErrUnexpectedEOF, + }, + { // malformed length + have: "badbeef", + err: io.ErrUnexpectedEOF, + }, + { // specified length longer than actual value + have: "beef010000000000", + err: io.ErrUnexpectedEOF, + }, + } { + r := NewReader(io.NopCloser(bytes.NewBuffer(common.Hex2Bytes(tt.have)))) + if tt.err != nil { + if _, err := r.Read(); err != nil && tt.err != nil && err.Error() != tt.err.Error() { + t.Fatalf("expected error %v, got %v", tt.err, err) + } + continue + } + for _, want := range tt.want { + if got, err := r.Read(); err != nil { + t.Fatalf("test %d: decoding error: %v", i, err) + } else if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) { + t.Fatalf("test %d: decoded entry does not match (want %v, got %v)", i, want, got) + } + } + } +} + +// WriteSeeker is an in-memory io.Writer and io.Seeker implementation. +type WriteSeeker struct { + pos int64 + buf []byte +} + +func NewWriteSeeker() *WriteSeeker { + return &WriteSeeker{} +} + +func (w *WriteSeeker) Write(p []byte) (n int, err error) { + if len(w.buf) != int(w.pos) { + return 0, fmt.Errorf("writing after seek not supported") + } + w.buf = append(w.buf, p...) + w.pos += int64(len(p)) + return len(p), nil +} + +func (w *WriteSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + w.pos = offset + case io.SeekCurrent: + w.pos = w.pos + offset + case io.SeekEnd: + w.pos = int64(len(w.buf)) + offset + default: + return 0, fmt.Errorf("unknown seek whence %d", whence) + } + if w.pos < 0 { + w.pos = 0 + } + return w.pos, nil +} + +func (w *WriteSeeker) Bytes() []byte { + return w.buf +} diff --git a/internal/era/era.go b/internal/era/era.go new file mode 100644 index 0000000000..3ae68b8279 --- /dev/null +++ b/internal/era/era.go @@ -0,0 +1,528 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + "os" + "path" + "strconv" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +var ( + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x03 + TypeCompressedBody uint16 = 0x04 + TypeCompressedReceipts uint16 = 0x05 + TypeTotalDifficulty uint16 = 0x06 + TypeAccumulator uint16 = 0x07 + TypeBlockIndex uint16 = 0x3266 + + MaxEra1Size = 8192 +) + +// Filename returns a recognizable Era1-formatted file name for the specified +// epoch and network. +func Filename(network string, epoch int, root common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10]) +} + +// ReadDir reads all the era1 files in a directory for a given network. +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) + } + var ( + next = uint64(0) + eras []string + ) + for _, entry := range entries { + if path.Ext(entry.Name()) == ".era1" { + n := strings.Split(entry.Name(), "-") + if len(n) != 3 { + // invalid era1 filename, skip + continue + } + if n[0] == network { + if epoch, err := strconv.ParseUint(n[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + next += 1 + eras = append(eras, entry.Name()) + } + } + } + return eras, nil +} + +// Builder is used to create Era1 archives of block data. +// +// Era1 files are themselves e2store files. For more information on this format, +// see https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md. +// +// The overall structure of an Era1 file follows closely the structure of an Era file +// which contains consensus Layer data (and as a byproduct, EL data after the merge). +// +// The structure can be summarized through this definition: +// +// era1 := Version | block-tuple* | other-entries* | Accumulator | BlockIndex +// block-tuple := CompressedHeader | CompressedBody | CompressedReceipts | TotalDifficulty +// +// Each basic element is its own entry: +// +// Version = { type: [0x65, 0x32], data: nil } +// CompressedHeader = { type: [0x03, 0x00], data: snappyFramed(rlp(header)) } +// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) } +// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) } +// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) } +// Accumulator = { type: [0x07, 0x00], data: accumulator-root } +// BlockIndex = { type: [0x32, 0x66], data: block-index } +// +// Accumulator is computed by constructing an SSZ list of header-records of length at most +// 8192 and then calculating the hash_tree_root of that list. +// +// header-record := { block-hash: Bytes32, total-difficulty: Uint256 } +// accumulator := hash_tree_root([]header-record, 8192) +// +// BlockIndex stores relative offsets to each compressed block entry. The +// format is: +// +// block-index := starting-number | index | index | index ... | count +// +// starting-number is the first block number in the archive. Every index is a +// defined relative to index's location in the file. The total number of block +// entries in the file is recorded in count. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in +// an Era1 batch is also 8192. +type Builder struct { + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + written int +} + +// NewBuilder returns a new Builder instance. +func NewBuilder(w io.Writer) *Builder { + return &Builder{ + w: e2store.NewWriter(w), + hashes: make([]common.Hash, 0), + tds: make([]*big.Int, 0), + } +} + +// Add writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { + eh, err := rlp.EncodeToBytes(block.Header()) + if err != nil { + return err + } + eb, err := rlp.EncodeToBytes(block.Body()) + if err != nil { + return err + } + er, err := rlp.EncodeToBytes(receipts) + if err != nil { + return err + } + return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty()) +} + +// AddRLP writes a compressed block entry and compressed receipts entry to the +// underlying e2store file. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { + // Write Era1 version entry before first block. + if b.startNum == nil { + if err := writeVersion(b.w); err != nil { + return err + } + n := number + b.startNum = &n + b.startTd = new(big.Int).Sub(td, difficulty) + } + if len(b.indexes) >= MaxEra1Size { + return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) + } + + b.indexes = append(b.indexes, uint64(b.written)) + b.hashes = append(b.hashes, hash) + b.tds = append(b.tds, td) + + // Small helper to take care snappy encoding and writing e2store entry. + snappyWrite := func(typ uint16, in []byte) error { + var ( + buf = bytes.NewBuffer(nil) + s = snappy.NewBufferedWriter(buf) + ) + if _, err := s.Write(in); err != nil { + return fmt.Errorf("error snappy encoding: %w", err) + } + if err := s.Flush(); err != nil { + return fmt.Errorf("error flushing snappy encoding: %w", err) + } + n, err := b.w.Write(typ, buf.Bytes()) + b.written += n + if err != nil { + return fmt.Errorf("error writing e2store entry: %w", err) + } + return nil + } + + // Write block data. + if err := snappyWrite(TypeCompressedHeader, header); err != nil { + return err + } + if err := snappyWrite(TypeCompressedBody, body); err != nil { + return err + } + if err := snappyWrite(TypeCompressedReceipts, receipts); err != nil { + return err + } + // Also write total difficulty, but don't snappy encode. + btd := bigToBytes32(td) + n, err := b.w.Write(TypeTotalDifficulty, btd[:]) + b.written += n + if err != nil { + return err + } + + return nil +} + +// Finalize computes the accumulator and block index values, then writes the +// corresponding e2store entries. +func (b *Builder) Finalize() (common.Hash, error) { + if b.startNum == nil { + return common.Hash{}, fmt.Errorf("finalize called on empty builder") + } + // Compute accumulator root and write entry. + root, err := ComputeAccumulator(b.hashes, b.tds) + if err != nil { + return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err) + } + n, err := b.w.Write(TypeAccumulator, root[:]) + b.written += n + if err != nil { + return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) + } + // Get beginning of index entry to calculate block relative offset. + base := int64(b.written + (3 * 8)) // skip e2store header (type, length) and start block + + // Construct block index. Detailed format described in Builder + // documentation, but it is essentially encoded as: + // "start | index | index | ... | count" + var ( + count = len(b.indexes) + index = make([]byte, 16+count*8) + ) + binary.LittleEndian.PutUint64(index, *b.startNum) + // Each offset is relative from the position it is encoded in the + // index. This means that even if the same block was to be included in + // the index twice (this would be invalid anyways), the relative offset + // would be different. The idea with this is that after reading a + // relative offset, the corresponding block can be quickly read by + // performing a seek relative to the current position. + for i, offset := range b.indexes { + relative := int64(offset) - (base + int64(i)*8) + binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative)) + } + binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) + + // Finally, write the block index entry. + if _, err := b.w.Write(TypeBlockIndex, index); err != nil { + return common.Hash{}, fmt.Errorf("unable to write block index: %w", err) + } + + return root, nil +} + +// writeVersion writes a version entry to e2store. +func writeVersion(w *e2store.Writer) error { + _, err := w.Write(TypeVersion, nil) + return err +} + +// Reader reads an Era1 archive. +// See Builder documentation for a detailed explanation of the Era1 format. +type Reader struct { + r io.ReadSeeker + offset uint64 + metadata metadata +} + +// NewReader returns a new Reader instance. +func NewReader(r io.ReadSeeker) (*Reader, error) { + m, err := readMetadata(r) + if err != nil { + return nil, err + } + return &Reader{r, m.start, m}, nil +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. It is normalized against the index's +// start block. +func (r *Reader) readOffset(n uint64) (int64, error) { + // Seek to the encoding of the block's offset. + var ( + firstIndex = -8 - int64(r.metadata.count)*8 // size of count - index entries + indexOffset = int64(n-r.metadata.start) * 8 // desired index * size of indexes + ) + if _, err := r.r.Seek(firstIndex+indexOffset, io.SeekEnd); err != nil { + return 0, err + } + // Read the block's offset. + var offset int64 + if err := binary.Read(r.r, binary.LittleEndian, &offset); err != nil { + return 0, err + } + return offset, nil +} + +// Read reads one (block, receipts) tuple from an Era1 archive. +func (r *Reader) Read() (*types.Block, types.Receipts, error) { + block, receipts, err := r.ReadBlockAndReceipts(r.offset) + if err != nil { + return nil, nil, err + } + r.offset += 1 + return block, receipts, nil +} + +// ReadHeader reads the header number n RLP. +func (r *Reader) ReadHeaderRLP(n uint64) ([]byte, error) { + // Determine if the request can served by current the Era1 file, e.g. n + // must be within the range of blocks specified in the block index + // metadata. + if n < r.metadata.start || r.metadata.start+r.metadata.count < n { + return nil, fmt.Errorf("request out-of-bounds: want %d, start: %d, count: %d", n, r.metadata.start, r.metadata.count) + } + // Read the specified block's offset from the block index. + offset, err := r.readOffset(n) + if err != nil { + return nil, fmt.Errorf("error reading block offset: %w", err) + } + if _, err := r.r.Seek(offset, io.SeekCurrent); err != nil { + return nil, err + } + // Read header. + entry, err := e2store.NewReader(r.r).Read() + if err != nil { + return nil, err + } + if entry.Type != TypeCompressedHeader { + return nil, fmt.Errorf("expected header entry, got %x", entry.Type) + } + return io.ReadAll(snappy.NewReader(bytes.NewReader(entry.Value))) +} + +// ReadBodyRLP reads the block body number n RLP. +func (r *Reader) ReadBodyRLP(n uint64) ([]byte, error) { + // Orient cursor. + _, err := r.ReadHeaderRLP(n) + if err != nil { + return nil, err + } + // Read body. + entry, err := e2store.NewReader(r.r).Read() + if err != nil { + return nil, err + } + if entry.Type != TypeCompressedBody { + return nil, fmt.Errorf("expected body entry, got %x", entry.Type) + } + return io.ReadAll(snappy.NewReader(bytes.NewReader(entry.Value))) +} + +// ReadReceiptsRLP reads the receipts RLP associated with number n. +func (r *Reader) ReadReceiptsRLP(n uint64) ([]byte, error) { + // Orient cursor. + _, err := r.ReadBodyRLP(n) + if err != nil { + return nil, err + } + // Read receipts. + entry, err := e2store.NewReader(r.r).Read() + if err != nil { + return nil, err + } + if entry.Type != TypeCompressedReceipts { + return nil, fmt.Errorf("expected receipts entry, got %x", entry.Type) + } + return io.ReadAll(snappy.NewReader(bytes.NewReader(entry.Value))) +} + +// ReadTotalDifficulty reads the total difficulty of block number n. +func (r *Reader) ReadTotalDifficulty(n uint64) (*big.Int, error) { + // Orient cursor. + _, err := r.ReadReceiptsRLP(n) + if err != nil { + return nil, err + } + // Read totaly difficulty. + entry, err := e2store.NewReader(r.r).Read() + if err != nil { + return nil, err + } + if entry.Type != TypeTotalDifficulty { + return nil, fmt.Errorf("expected total difficulty entry, got %x", entry.Type) + } + return new(big.Int).SetBytes(reverseOrder(entry.Value)), nil +} + +// ReadHeader reads the header number n. +func (r *Reader) ReadHeader(n uint64) (*types.Header, error) { + h, err := r.ReadHeaderRLP(n) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.DecodeBytes(h, &header); err != nil { + return nil, err + } + return &header, nil +} + +// ReadBlock reads the block number n. +func (r *Reader) ReadBlock(n uint64) (*types.Block, error) { + header, err := r.ReadHeader(n) + if err != nil { + return nil, err + } + b, err := r.ReadBodyRLP(n) + if err != nil { + return nil, err + } + var body types.Body + if err := rlp.DecodeBytes(b, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles), nil +} + +// ReadBlockAndReceipts reads the block number n and associated receipts. +func (r *Reader) ReadBlockAndReceipts(n uint64) (*types.Block, types.Receipts, error) { + // Read block. + block, err := r.ReadBlock(n) + if err != nil { + return nil, nil, err + } + // Read receipts. + rr, err := r.ReadReceiptsRLP(n) + if err != nil { + return nil, nil, err + } + // Decode receipts. + var receipts types.Receipts + if err := rlp.DecodeBytes(rr, &receipts); err != nil { + return nil, nil, err + } + return block, receipts, err +} + +// Accumulator reads the accumulator entry in the Era1 file. +func (r *Reader) Accumulator() (common.Hash, error) { + _, err := r.seek(0, io.SeekStart) + if err != nil { + return common.Hash{}, err + } + entry, err := e2store.NewReader(r.r).Find(TypeAccumulator) + if err != nil { + return common.Hash{}, err + } + return common.BytesToHash(entry.Value), nil +} + +// InitialTD returns initial total difficulty before the difficulty of the +// first block of the Era1 is applied. +func (r *Reader) InitialTD() (*big.Int, error) { + _, err := r.seek(0, io.SeekStart) + if err != nil { + return nil, err + } + h, err := r.ReadHeader(r.Start()) + if err != nil { + return nil, err + } + // Above seek also sets reader so next TD entry will be for this block. + entry, err := e2store.NewReader(r.r).Find(TypeTotalDifficulty) + if err != nil { + return nil, err + } + td := new(big.Int).SetBytes(reverseOrder(entry.Value)) + return td.Sub(td, h.Difficulty), nil +} + +// Start returns the listed start block. +func (r *Reader) Start() uint64 { + return r.metadata.start +} + +// Count returns the total number of blocks in the Era1. +func (r *Reader) Count() uint64 { + return r.metadata.count +} + +// seek is a shorthand method for calling seek on the inner reader. +func (r *Reader) seek(offset int64, whence int) (int64, error) { + return r.r.Seek(offset, whence) +} + +// metadata wraps the metadata in the block index. +type metadata struct { + start, count uint64 +} + +// readMetadata reads the metadata stored in an Era1 file's block index. +func readMetadata(r io.ReadSeeker) (m metadata, err error) { + // Seek to count value. It's the last 8 bytes of the file. + if _, err = r.Seek(-8, io.SeekEnd); err != nil { + return + } + // Read count. + if err = binary.Read(r, binary.LittleEndian, &m.count); err != nil { + return + } + // Seek to start value. It's at the offset -sizeof(m.count) - + // count*sizeof(indexEntry) - sizeof(m.start) + if _, err = r.Seek(-16-int64(m.count)*8, io.SeekEnd); err != nil { + return + } + // Read start. + if err = binary.Read(r, binary.LittleEndian, &m.start); err != nil { + return + } + return +} diff --git a/internal/era/era_test.go b/internal/era/era_test.go new file mode 100644 index 0000000000..1c646fa47f --- /dev/null +++ b/internal/era/era_test.go @@ -0,0 +1,132 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "bytes" + "math/big" + "os" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +type testchain struct { + headers [][]byte + bodies [][]byte + receipts [][]byte + tds []*big.Int +} + +func TestEra1Builder(t *testing.T) { + // Get temp directory. + f, err := os.CreateTemp("", "era1-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + var ( + builder = NewBuilder(f) + chain = testchain{} + ) + for i := 0; i < 128; i++ { + chain.headers = append(chain.headers, []byte{byte('h'), byte(i)}) + chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)}) + chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)}) + chain.tds = append(chain.tds, big.NewInt(int64(i))) + } + + // Write blocks to Era1. + for i := 0; i < len(chain.headers); i++ { + var ( + header = chain.headers[i] + body = chain.bodies[i] + receipts = chain.receipts[i] + hash = common.Hash{byte(i)} + td = chain.tds[i] + ) + if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil { + t.Fatalf("error adding entry: %v", err) + } + } + + // Finalize Era1. + if _, err := builder.Finalize(); err != nil { + t.Fatalf("error finalizing era1: %v", err) + } + + // Verify Era1 contents. + r, err := NewReader(f) + if err != nil { + t.Fatalf("failed to make reader: %s", err) + } + for i := uint64(0); i < uint64(len(chain.headers)); i++ { + // Check headers. + header, err := r.ReadHeaderRLP(i) + if err != nil { + t.Fatalf("error reading from era1: %v", err) + } + if !bytes.Equal(header, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) + } + + // Check bodies. + body, err := r.ReadBodyRLP(i) + if err != nil { + t.Fatalf("error reading from era1: %v", err) + } + if !bytes.Equal(body, chain.bodies[i]) { + t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) + } + + // Check receipts. + receipts, err := r.ReadReceiptsRLP(i) + if err != nil { + t.Fatalf("error reading from era1: %v", err) + } + if !bytes.Equal(receipts, chain.receipts[i]) { + t.Fatalf("mismatched body: want %s, got %s", chain.receipts[i], receipts) + } + + // Check total difficulty. + td, err := r.ReadTotalDifficulty(i) + if err != nil { + t.Fatalf("error reading from era1: %v", err) + } + if td.Cmp(chain.tds[i]) != 0 { + t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) + } + } +} + +func TestEraFilename(t *testing.T) { + for i, tt := range []struct { + network string + epoch int + root common.Hash + expected string + }{ + {"mainnet", 1, common.Hash{1}, "mainnet-00001-01000000.era1"}, + {"goerli", 99999, common.HexToHash("0xdeadbeef00000000000000000000000000000000000000000000000000000000"), "goerli-99999-deadbeef.era1"}, + } { + got := Filename(tt.network, tt.epoch, tt.root) + if tt.expected != got { + t.Errorf("test %d: invalid filename: want %s, got %s", i, tt.expected, got) + } + } +}