mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-02-26 07:37:20 +00:00
cmd/geth, internal/era/eradl: add era1 downloader tool (#31823)
This adds a geth subcommand for downloading era1 files and placing them into the correct location. The tool can be used even while geth is already running on the datadir. Downloads are checked against a hard-coded list of checksums for mainnet and sepolia. ``` ./geth download-era --server $SERVER --block 333333 ./geth download-era --server $SERVER --block 333333-444444 ./geth download-era --server $SERVER --epoch 0-10 ./geth download-era --server $SERVER --all ``` The implementation reuses the file downloader we already had for fetching build tools. I've done some refactoring on it to make sure it can support the new use case, and there are some changes to the build here as well.
This commit is contained in:
Felix Lange
GitHub
parent
85b26f3d68
commit
3ceec0ea9b
9 changed files with 2642 additions and 236 deletions
55
build/ci.go
55
build/ci.go
|
|
@ -59,6 +59,7 @@ import (
|
|||
"github.com/cespare/cp"
|
||||
"github.com/ethereum/go-ethereum/crypto/signify"
|
||||
"github.com/ethereum/go-ethereum/internal/build"
|
||||
"github.com/ethereum/go-ethereum/internal/download"
|
||||
"github.com/ethereum/go-ethereum/internal/version"
|
||||
)
|
||||
|
||||
|
|
@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
|
|||
// Configure the toolchain.
|
||||
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
|
||||
if *dlgo {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
tc.Root = build.DownloadGo(csdb)
|
||||
}
|
||||
// Disable CLI markdown doc generation in release builds.
|
||||
|
|
@ -285,7 +286,7 @@ func doTest(cmdline []string) {
|
|||
flag.CommandLine.Parse(cmdline)
|
||||
|
||||
// Get test fixtures.
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
downloadSpecTestFixtures(csdb, *cachedir)
|
||||
|
||||
// Configure the toolchain.
|
||||
|
|
@ -329,16 +330,11 @@ func doTest(cmdline []string) {
|
|||
}
|
||||
|
||||
// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
|
||||
func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
|
||||
executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
|
||||
ext := ".tar.gz"
|
||||
base := "fixtures_pectra-devnet-6" // TODO(s1na) rename once the version becomes part of the filename
|
||||
url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/%s/%s%s", executionSpecTestsVersion, base, ext)
|
||||
archivePath := filepath.Join(cachedir, base+ext)
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
|
||||
|
|
@ -444,14 +440,13 @@ func doLint(cmdline []string) {
|
|||
|
||||
// downloadLinter downloads and unpacks golangci-lint.
|
||||
func downloadLinter(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "golangci")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("golangci")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
arch := runtime.GOARCH
|
||||
ext := ".tar.gz"
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
ext = ".zip"
|
||||
}
|
||||
|
|
@ -459,9 +454,8 @@ func downloadLinter(cachedir string) string {
|
|||
arch += "v" + os.Getenv("GOARM")
|
||||
}
|
||||
base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
|
||||
url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
|
||||
archivePath := filepath.Join(cachedir, base+ext)
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := build.ExtractArchive(archivePath, cachedir); err != nil {
|
||||
|
|
@ -497,8 +491,8 @@ func protocArchiveBaseName() (string, error) {
|
|||
// in the generate command. It returns the full path of the directory
|
||||
// containing the 'protoc-gen-go' executable.
|
||||
func downloadProtocGenGo(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "protoc-gen-go")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("protoc-gen-go")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -510,10 +504,8 @@ func downloadProtocGenGo(cachedir string) string {
|
|||
archiveName += ".tar.gz"
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName)
|
||||
|
||||
archivePath := path.Join(cachedir, archiveName)
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest := filepath.Join(cachedir, baseName)
|
||||
|
|
@ -531,8 +523,8 @@ func downloadProtocGenGo(cachedir string) string {
|
|||
// files as a CI step. It returns the full path to the directory containing
|
||||
// the protoc executable.
|
||||
func downloadProtoc(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "protoc")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("protoc")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -543,10 +535,8 @@ func downloadProtoc(cachedir string) string {
|
|||
|
||||
fileName := fmt.Sprintf("protoc-%s-%s", version, baseName)
|
||||
archiveFileName := fileName + ".zip"
|
||||
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName)
|
||||
archivePath := filepath.Join(cachedir, archiveFileName)
|
||||
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest := filepath.Join(cachedir, fileName)
|
||||
|
|
@ -826,18 +816,17 @@ func doDebianSource(cmdline []string) {
|
|||
// downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
|
||||
// to bootstrap the builder Go.
|
||||
func downloadGoBootstrapSources(cachedir string) []string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
|
||||
var bundles []string
|
||||
for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
|
||||
gobootVersion, err := build.Version(csdb, booter)
|
||||
gobootVersion, err := csdb.FindVersion(booter)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
|
||||
url := "https://dl.google.com/go/" + file
|
||||
dst := filepath.Join(cachedir, file)
|
||||
if err := csdb.DownloadFile(url, dst); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
bundles = append(bundles, dst)
|
||||
|
|
@ -847,15 +836,14 @@ func downloadGoBootstrapSources(cachedir string) []string {
|
|||
|
||||
// downloadGoSources downloads the Go source tarball.
|
||||
func downloadGoSources(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
dlgoVersion, err := build.Version(csdb, "golang")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
dlgoVersion, err := csdb.FindVersion("golang")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
|
||||
url := "https://dl.google.com/go/" + file
|
||||
dst := filepath.Join(cachedir, file)
|
||||
if err := csdb.DownloadFile(url, dst); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return dst
|
||||
|
|
@ -1181,5 +1169,6 @@ func doPurge(cmdline []string) {
|
|||
}
|
||||
|
||||
func doSanityCheck() {
|
||||
build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
csdb.DownloadAndVerifyAll()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,9 +21,12 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
|
|
@ -39,6 +42,7 @@ import (
|
|||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/internal/debug"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/eradl"
|
||||
"github.com/ethereum/go-ethereum/internal/flags"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
|
|
@ -190,7 +194,7 @@ This command dumps out the state for a given block (or latest, if none provided)
|
|||
`,
|
||||
}
|
||||
|
||||
pruneCommand = &cli.Command{
|
||||
pruneHistoryCommand = &cli.Command{
|
||||
Action: pruneHistory,
|
||||
Name: "prune-history",
|
||||
Usage: "Prune blockchain history (block bodies and receipts) up to the merge block",
|
||||
|
|
@ -201,6 +205,42 @@ The prune-history command removes historical block bodies and receipts from the
|
|||
blockchain database up to the merge block, while preserving block headers. This
|
||||
helps reduce storage requirements for nodes that don't need full historical data.`,
|
||||
}
|
||||
|
||||
downloadEraCommand = &cli.Command{
|
||||
Action: downloadEra,
|
||||
Name: "download-era",
|
||||
Usage: "Fetches era1 files (pre-merge history) from an HTTP endpoint",
|
||||
ArgsUsage: "",
|
||||
Flags: slices.Concat(
|
||||
utils.DatabaseFlags,
|
||||
utils.NetworkFlags,
|
||||
[]cli.Flag{
|
||||
eraBlockFlag,
|
||||
eraEpochFlag,
|
||||
eraAllFlag,
|
||||
eraServerFlag,
|
||||
},
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
var (
|
||||
eraBlockFlag = &cli.StringFlag{
|
||||
Name: "block",
|
||||
Usage: "Block number to fetch. (can also be a range <start>-<end>)",
|
||||
}
|
||||
eraEpochFlag = &cli.StringFlag{
|
||||
Name: "epoch",
|
||||
Usage: "Epoch number to fetch (can also be a range <start>-<end>)",
|
||||
}
|
||||
eraAllFlag = &cli.BoolFlag{
|
||||
Name: "all",
|
||||
Usage: "Download all available era1 files",
|
||||
}
|
||||
eraServerFlag = &cli.StringFlag{
|
||||
Name: "server",
|
||||
Usage: "era1 server URL",
|
||||
}
|
||||
)
|
||||
|
||||
// initGenesis will initialise the given JSON format genesis file and writes it as
|
||||
|
|
@ -665,3 +705,83 @@ func pruneHistory(ctx *cli.Context) error {
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
// downladEra is the era1 file downloader tool.
|
||||
func downloadEra(ctx *cli.Context) error {
|
||||
flags.CheckExclusive(ctx, eraBlockFlag, eraEpochFlag, eraAllFlag)
|
||||
|
||||
// Resolve the network.
|
||||
var network = "mainnet"
|
||||
if utils.IsNetworkPreset(ctx) {
|
||||
switch {
|
||||
case ctx.IsSet(utils.MainnetFlag.Name):
|
||||
case ctx.IsSet(utils.SepoliaFlag.Name):
|
||||
network = "sepolia"
|
||||
default:
|
||||
return fmt.Errorf("unsupported network, no known era1 checksums")
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the destination directory.
|
||||
stack, _ := makeConfigNode(ctx)
|
||||
defer stack.Close()
|
||||
ancients := stack.ResolveAncient("chaindata", "")
|
||||
dir := filepath.Join(ancients, "era")
|
||||
|
||||
baseURL := ctx.String(eraServerFlag.Name)
|
||||
if baseURL == "" {
|
||||
return fmt.Errorf("need --%s flag to download", eraServerFlag.Name)
|
||||
}
|
||||
|
||||
l, err := eradl.New(baseURL, network)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case ctx.IsSet(eraAllFlag.Name):
|
||||
return l.DownloadAll(dir)
|
||||
|
||||
case ctx.IsSet(eraBlockFlag.Name):
|
||||
s := ctx.String(eraBlockFlag.Name)
|
||||
start, end, ok := parseRange(s)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid block range: %q", s)
|
||||
}
|
||||
return l.DownloadBlockRange(start, end, dir)
|
||||
|
||||
case ctx.IsSet(eraEpochFlag.Name):
|
||||
s := ctx.String(eraEpochFlag.Name)
|
||||
start, end, ok := parseRange(s)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid epoch range: %q", s)
|
||||
}
|
||||
return l.DownloadEpochRange(start, end, dir)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("specify one of --%s, --%s, or --%s to download", eraAllFlag.Name, eraBlockFlag.Name, eraEpochFlag.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func parseRange(s string) (start uint64, end uint64, ok bool) {
|
||||
if m, _ := regexp.MatchString("[0-9]+", s); m {
|
||||
start, err := strconv.ParseUint(s, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
end = start
|
||||
return start, end, true
|
||||
}
|
||||
if m, _ := regexp.MatchString("[0-9]+-[0-9]+", s); m {
|
||||
s1, s2, _ := strings.Cut(s, "-")
|
||||
start, err := strconv.ParseUint(s1, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
end, err = strconv.ParseUint(s2, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, false
|
||||
}
|
||||
return start, end, true
|
||||
}
|
||||
return 0, 0, false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -225,7 +225,8 @@ func init() {
|
|||
removedbCommand,
|
||||
dumpCommand,
|
||||
dumpGenesisCommand,
|
||||
pruneCommand,
|
||||
pruneHistoryCommand,
|
||||
downloadEraCommand,
|
||||
// See accountcmd.go:
|
||||
accountCommand,
|
||||
walletCommand,
|
||||
|
|
|
|||
|
|
@ -1,151 +0,0 @@
|
|||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package build
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ChecksumDB keeps file checksums.
|
||||
type ChecksumDB struct {
|
||||
allChecksums []string
|
||||
}
|
||||
|
||||
// MustLoadChecksums loads a file containing checksums.
|
||||
func MustLoadChecksums(file string) *ChecksumDB {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Fatal("can't load checksum file: " + err.Error())
|
||||
}
|
||||
return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
|
||||
}
|
||||
|
||||
// Verify checks whether the given file is valid according to the checksum database.
|
||||
func (db *ChecksumDB) Verify(path string) error {
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash := hex.EncodeToString(h.Sum(nil))
|
||||
if !db.findHash(filepath.Base(path), fileHash) {
|
||||
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *ChecksumDB) findHash(basename, hash string) bool {
|
||||
want := hash + " " + basename
|
||||
for _, line := range db.allChecksums {
|
||||
if strings.TrimSpace(line) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DownloadFile downloads a file and verifies its checksum.
|
||||
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
|
||||
if err := db.Verify(dstPath); err == nil {
|
||||
fmt.Printf("%s is up-to-date\n", dstPath)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s is stale\n", dstPath)
|
||||
fmt.Printf("downloading from %s\n", url)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download error: status %d", resp.StatusCode)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := newDownloadWriter(fd, resp.ContentLength)
|
||||
_, err = io.Copy(dst, resp.Body)
|
||||
dst.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Verify(dstPath)
|
||||
}
|
||||
|
||||
type downloadWriter struct {
|
||||
file *os.File
|
||||
dstBuf *bufio.Writer
|
||||
size int64
|
||||
written int64
|
||||
lastpct int64
|
||||
}
|
||||
|
||||
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
|
||||
return &downloadWriter{
|
||||
file: dst,
|
||||
dstBuf: bufio.NewWriter(dst),
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Write(buf []byte) (int, error) {
|
||||
n, err := w.dstBuf.Write(buf)
|
||||
|
||||
// Report progress.
|
||||
w.written += int64(n)
|
||||
pct := w.written * 10 / w.size * 10
|
||||
if pct != w.lastpct {
|
||||
if w.lastpct != 0 {
|
||||
fmt.Print("...")
|
||||
}
|
||||
fmt.Print(pct, "%")
|
||||
w.lastpct = pct
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Close() error {
|
||||
if w.lastpct > 0 {
|
||||
fmt.Println() // Finish the progress line.
|
||||
}
|
||||
flushErr := w.dstBuf.Flush()
|
||||
closeErr := w.file.Close()
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
|
|
@ -24,6 +24,8 @@ import (
|
|||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/internal/download"
|
||||
)
|
||||
|
||||
type GoToolchain struct {
|
||||
|
|
@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
|
|||
|
||||
// DownloadGo downloads the Go binary distribution and unpacks it into a temporary
|
||||
// directory. It returns the GOROOT of the unpacked toolchain.
|
||||
func DownloadGo(csdb *ChecksumDB) string {
|
||||
version, err := Version(csdb, "golang")
|
||||
func DownloadGo(csdb *download.ChecksumDB) string {
|
||||
version, err := csdb.FindVersion("golang")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
|
|||
}
|
||||
return goroot
|
||||
}
|
||||
|
||||
// Version returns the versions defined in the checksumdb.
|
||||
func Version(csdb *ChecksumDB, version string) (string, error) {
|
||||
for _, l := range csdb.allChecksums {
|
||||
if !strings.HasPrefix(l, "# version:") {
|
||||
continue
|
||||
}
|
||||
v := strings.Split(l, ":")[1]
|
||||
parts := strings.Split(v, " ")
|
||||
if len(parts) != 2 {
|
||||
log.Print("Erroneous version-string", "v", l)
|
||||
continue
|
||||
}
|
||||
if parts[0] == version {
|
||||
return parts[1], nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no version found for '%v'", version)
|
||||
}
|
||||
|
||||
// DownloadAndVerifyChecksums downloads all files and checks that they match
|
||||
// the checksum given in checksums.txt.
|
||||
// This task can be used to sanity-check new checksums.
|
||||
func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
|
||||
var (
|
||||
base = ""
|
||||
ucache = os.TempDir()
|
||||
)
|
||||
for _, l := range csdb.allChecksums {
|
||||
if strings.HasPrefix(l, "# https://") {
|
||||
base = l[2:]
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(l, "#") {
|
||||
continue
|
||||
}
|
||||
hashFile := strings.Split(l, " ")
|
||||
if len(hashFile) != 2 {
|
||||
continue
|
||||
}
|
||||
file := hashFile[1]
|
||||
url := base + file
|
||||
dst := filepath.Join(ucache, file)
|
||||
if err := csdb.DownloadFile(url, dst); err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
298
internal/download/download.go
Normal file
298
internal/download/download.go
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package download implements checksum-verified file downloads.
|
||||
package download
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"iter"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ChecksumDB keeps file checksums and tool versions.
|
||||
type ChecksumDB struct {
|
||||
hashes []hashEntry
|
||||
versions []versionEntry
|
||||
}
|
||||
|
||||
type versionEntry struct {
|
||||
name string
|
||||
version string
|
||||
}
|
||||
|
||||
type hashEntry struct {
|
||||
hash string
|
||||
file string
|
||||
url *url.URL
|
||||
}
|
||||
|
||||
// MustLoadChecksums loads a file containing checksums.
|
||||
func MustLoadChecksums(file string) *ChecksumDB {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
panic("can't load checksum file: " + err.Error())
|
||||
}
|
||||
db, err := ParseChecksums(content)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("invalid checksums in %s: %v", file, err))
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// ParseChecksums parses a checksum database.
|
||||
func ParseChecksums(input []byte) (*ChecksumDB, error) {
|
||||
var (
|
||||
csdb = new(ChecksumDB)
|
||||
rd = bytes.NewBuffer(input)
|
||||
lastURL *url.URL
|
||||
)
|
||||
for lineNum := 1; ; lineNum++ {
|
||||
line, err := rd.ReadString('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case line == "":
|
||||
// Blank lines are allowed, and they reset the current urlEntry.
|
||||
lastURL = nil
|
||||
|
||||
case strings.HasPrefix(line, "#"):
|
||||
// It's a comment. Some comments have special meaning.
|
||||
content := strings.TrimLeft(line, "# ")
|
||||
switch {
|
||||
case strings.HasPrefix(content, "version:"):
|
||||
// Version comments define the version of a tool.
|
||||
v := strings.Split(content, ":")[1]
|
||||
parts := strings.Split(v, " ")
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
|
||||
}
|
||||
csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
|
||||
|
||||
case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
|
||||
// URL comments define the URL where the following files are found. Here
|
||||
// we keep track of the last found urlEntry and attach it to each file later.
|
||||
u, err := url.Parse(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: invalid URL: %v", lineNum, err)
|
||||
}
|
||||
lastURL = u
|
||||
}
|
||||
|
||||
default:
|
||||
// It's a file hash entry.
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
|
||||
}
|
||||
csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
|
||||
}
|
||||
}
|
||||
return csdb, nil
|
||||
}
|
||||
|
||||
// Files returns an iterator over all file names.
|
||||
func (db *ChecksumDB) Files() iter.Seq[string] {
|
||||
return func(yield func(string) bool) {
|
||||
for _, e := range db.hashes {
|
||||
if !yield(e.file) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadAndVerifyAll downloads all files and checks that they match the checksum given in
|
||||
// the database. This task can be used to sanity-check new checksums.
|
||||
func (db *ChecksumDB) DownloadAndVerifyAll() {
|
||||
var tmp = os.TempDir()
|
||||
for _, e := range db.hashes {
|
||||
if e.url == nil {
|
||||
fmt.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
|
||||
continue
|
||||
}
|
||||
url := e.url.JoinPath(e.file).String()
|
||||
dst := filepath.Join(tmp, e.file)
|
||||
if err := db.DownloadFile(url, dst); err != nil {
|
||||
fmt.Println("error:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verifyHash checks that the file at 'path' has the expected hash.
|
||||
func verifyHash(path, expectedHash string) error {
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash := hex.EncodeToString(h.Sum(nil))
|
||||
if fileHash != expectedHash {
|
||||
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFileFromKnownURL downloads a file from the URL defined in the checksum database.
|
||||
func (db *ChecksumDB) DownloadFileFromKnownURL(dstPath string) error {
|
||||
base := filepath.Base(dstPath)
|
||||
url, err := db.FindURL(base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.DownloadFile(url, dstPath)
|
||||
}
|
||||
|
||||
// DownloadFile downloads a file and verifies its checksum.
|
||||
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
|
||||
basename := filepath.Base(dstPath)
|
||||
hash := db.findHash(basename)
|
||||
if hash == "" {
|
||||
return fmt.Errorf("no known hash for file %q", basename)
|
||||
}
|
||||
// Shortcut if already downloaded.
|
||||
if verifyHash(dstPath, hash) == nil {
|
||||
fmt.Printf("%s is up-to-date\n", dstPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%s is stale\n", dstPath)
|
||||
fmt.Printf("downloading from %s\n", url)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download error: status %d", resp.StatusCode)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Download to a temporary file.
|
||||
tmpfile := dstPath + ".tmp"
|
||||
fd, err := os.OpenFile(tmpfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := newDownloadWriter(fd, resp.ContentLength)
|
||||
_, err = io.Copy(dst, resp.Body)
|
||||
dst.Close()
|
||||
if err != nil {
|
||||
os.Remove(tmpfile)
|
||||
return err
|
||||
}
|
||||
if err := verifyHash(tmpfile, hash); err != nil {
|
||||
os.Remove(tmpfile)
|
||||
return err
|
||||
}
|
||||
// It's valid, rename to dstPath to complete the download.
|
||||
return os.Rename(tmpfile, dstPath)
|
||||
}
|
||||
|
||||
// findHash returns the known hash of a file.
|
||||
func (db *ChecksumDB) findHash(basename string) string {
|
||||
for _, e := range db.hashes {
|
||||
if e.file == basename {
|
||||
return e.hash
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FindVersion returns the current known version of a tool, if it is defined in the file.
|
||||
func (db *ChecksumDB) FindVersion(tool string) (string, error) {
|
||||
for _, e := range db.versions {
|
||||
if e.name == tool {
|
||||
return e.version, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
|
||||
}
|
||||
|
||||
// FindURL gets the URL for a file.
|
||||
func (db *ChecksumDB) FindURL(basename string) (string, error) {
|
||||
for _, e := range db.hashes {
|
||||
if e.file == basename {
|
||||
if e.url == nil {
|
||||
return "", fmt.Errorf("file %q has no URL defined", e.file)
|
||||
}
|
||||
return e.url.JoinPath(e.file).String(), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("file %q does not exist in checksum database", basename)
|
||||
}
|
||||
|
||||
type downloadWriter struct {
|
||||
file *os.File
|
||||
dstBuf *bufio.Writer
|
||||
size int64
|
||||
written int64
|
||||
lastpct int64
|
||||
}
|
||||
|
||||
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
|
||||
return &downloadWriter{
|
||||
file: dst,
|
||||
dstBuf: bufio.NewWriter(dst),
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Write(buf []byte) (int, error) {
|
||||
n, err := w.dstBuf.Write(buf)
|
||||
|
||||
// Report progress.
|
||||
w.written += int64(n)
|
||||
pct := w.written * 10 / w.size * 10
|
||||
if pct != w.lastpct {
|
||||
if w.lastpct != 0 {
|
||||
fmt.Print("...")
|
||||
}
|
||||
fmt.Print(pct, "%")
|
||||
w.lastpct = pct
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Close() error {
|
||||
if w.lastpct > 0 {
|
||||
fmt.Println() // Finish the progress line.
|
||||
}
|
||||
flushErr := w.dstBuf.Flush()
|
||||
closeErr := w.file.Close()
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
1897
internal/era/eradl/checksums_mainnet.txt
Normal file
1897
internal/era/eradl/checksums_mainnet.txt
Normal file
File diff suppressed because it is too large
Load diff
183
internal/era/eradl/checksums_sepolia.txt
Normal file
183
internal/era/eradl/checksums_sepolia.txt
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
ab7f6d4f4eba0267406f783b75accc7a93dece520242d04fed27b0af51d79242 sepolia-00000-643a00f7.era1
|
||||
9cae627459d13ed3d05f6812eefa93242555fbefd27aa0927b40b62d44feb2e9 sepolia-00001-f3ea493e.era1
|
||||
a6f691585bc74fae6c445a8985f0df342e983ba1ef569018befb4b21b30891e2 sepolia-00002-dcb01f4d.era1
|
||||
1add5a98a9e6c15a667d6a7bbdaea115893019f3033664c54c2dcab70829268d sepolia-00003-18b32584.era1
|
||||
5e5ce2ca04b0f1aef6f026214cb64d16f76606fb5b5baf8a462f4a851dda0513 sepolia-00004-e93b3f11.era1
|
||||
db6f7687e9826a4e4dbc1361d666a8f2aa735eae309f63d9cadf6b27a899277b sepolia-00005-90918472.era1
|
||||
f61118a4e1cb718bdb71fde1daf84e59c3524b1e241ea1a5e2112d8c53acc625 sepolia-00006-a4a583b1.era1
|
||||
fd0c6fc443a4c30617ce88a48e7438fecaf4cf4772034511fa51433c7aae181a sepolia-00007-ef1e0a86.era1
|
||||
95bc416c6393e0a579f9e4af42421cf861e43354d49273d369bcf9b38e070246 sepolia-00008-dfe1e6dd.era1
|
||||
8262e7182f9a8abb9f5fc05d745a3cd2061eacda5c87cc2533af2407bd20672c sepolia-00009-dba60c04.era1
|
||||
d7c123c8e06713df23db0046df7c45d873e153ff076357b6d0c5bc9b0081a246 sepolia-00010-c99c706d.era1
|
||||
7fd359358ebf710c5c61a6a65551a4cb91c2160dbd7b59ed355a4d7e10f29328 sepolia-00011-04513853.era1
|
||||
9d6ea741d1b91e1c2c37ac1659ef3516d4968b6e7c164d347e534c4bebe644fd sepolia-00012-6e07302d.era1
|
||||
ce3e580fd3e5cb9e0ba9ebffae948406fdbe59f96fa3a05a30a381bc7d1b583e sepolia-00013-25709be2.era1
|
||||
15df46dc8c2cfd402f302f2925d1c50ac12538d394e8687920f53346429d6743 sepolia-00014-7c90e5a5.era1
|
||||
1985aaeb96c81279b978beab1145b3baf460b12bff86b8ca05c5067e1b0ca25b sepolia-00015-fadc08ab.era1
|
||||
2a53126cd443e9310fa15180c5bbbdb76f257d7fee67159500d8efa0ed13bef4 sepolia-00016-d44b2499.era1
|
||||
e783cff42ba7a53461c7fcdaf59a5387e7091a1230aa6761af05e7daf59b4805 sepolia-00017-02451eb3.era1
|
||||
92848afbeea81397fc9790e7fba7c627c01c17cbc5b8961a48ca60329bbfa3f3 sepolia-00018-cdb53b11.era1
|
||||
d15ac188a4dfac75818dfeb04e72c2f269d9f985100a457ebdc17661e36f0d83 sepolia-00019-fa770019.era1
|
||||
e8ef40780f55a0b640d1b68e1a89d1d438e8105bb1388bffc8a743b88e357671 sepolia-00020-15fc80e7.era1
|
||||
8041d790a0c50044e331a385eb98b17f5050ab25637b6a4f2c3e8e99f3f81e6e sepolia-00021-b8814b14.era1
|
||||
f17adda073b66cc97cb58acc7b776296e135943718dab9fc7cfecbac748419b7 sepolia-00022-87944c43.era1
|
||||
159a5ec4fa1e79be6d13fe4753eaa8994f547e31783069082a39d51e3e00b178 sepolia-00023-516a30f5.era1
|
||||
5b75090c9f26899aa6fe504963b7f1a43ab13a8fd7547da42f9ca9cc980511de sepolia-00024-b922dde6.era1
|
||||
d5c39668b309b43eec720b2185ac23ef819cc7fa92e35f2e6c9a4ce9ee1c249d sepolia-00025-a081e175.era1
|
||||
188b5119e110feec067dc06aefde7decba2a87bde642cc1735391a1782a99b03 sepolia-00026-d3931a12.era1
|
||||
3114068f4eb9cf8277d4417da371e820f0bf1a654600f8dea149f9854523d1f2 sepolia-00027-43f5e0ac.era1
|
||||
b18597daf66f27976638a84a4cac5622f7d53fca8264dac3665f84f28435cc26 sepolia-00028-dae08170.era1
|
||||
d0672ee04c5a84e881c4acc3757e16f963c57b5b69da81a674498c1b06159f3e sepolia-00029-17818143.era1
|
||||
122dff18aaac0bae0a006da088794f638ee95ddaa1e0a347ca9a8eca816471c1 sepolia-00030-cc7fc4eb.era1
|
||||
42c2206fda093c7aa6b31fad1addf2dd14e862b4686b0e17050bd284dc479cfd sepolia-00031-3171eede.era1
|
||||
42b023d07a02ad739b61ec8297c43501ddac200d1866fa3203df466d2442c4b0 sepolia-00032-9e2ff5ca.era1
|
||||
2461fb005a9a62cf6ab52a0b3505abde9828c1f36def4fa12a547ebb5e8c89c0 sepolia-00033-bc921023.era1
|
||||
2ca97708fba029693d7754a85424d29f57ee296a48fcbde02597603e7e90525d sepolia-00034-b2496634.era1
|
||||
d2944e1cbd7a3276ca5ef9d4b4473557a65e58bd512029b2fb00d41f618ac551 sepolia-00035-f3add4f1.era1
|
||||
c7f63d7230dd7ae9c57c33014d56bd4c921f5db95f3872bf6745d03bc64b5cc0 sepolia-00036-0db12924.era1
|
||||
dc027011c543418fcfdd7897cea3356de16122f676da2459d78da5369cdc2b8f sepolia-00037-ae8bdd13.era1
|
||||
9be2262ef4e1e1b93cee3316ad473895bd7a7f3bff15ac8c18c1a157292ab7f6 sepolia-00038-81f80b03.era1
|
||||
56c03c86028c95d431030fa2728493ca8001cc86eec3f5d7a558224e550b42ef sepolia-00039-03e5d6f1.era1
|
||||
d64e65ecfcf480c2edbdebac350ac899b25874a522c226695a46fab17de5b15c sepolia-00040-296fe287.era1
|
||||
18226ba389f150581b36a234d877fad9c08c0fa691a366ba11f808ec9110cf67 sepolia-00041-5cd87470.era1
|
||||
0d2960f23933deadcdfe545fc8aba626e79c0c010fad0ac01b577ca5b99840e0 sepolia-00042-de86936c.era1
|
||||
b14f66c2fc74d0f772eb75c548a86da4bf160cf1067091c65c05a90812e7af1f sepolia-00043-ede4e682.era1
|
||||
15d704883fadfcbe4ce3520bc9ed5fcff98a8a419b6fa9f832394008ae1941aa sepolia-00044-4d49a81a.era1
|
||||
5343d8015eb0a6b8728b571fd0e7a2bb92728ec593469b0a9d8d2082a5afc36d sepolia-00045-7071c19b.era1
|
||||
63efe606b8c687aeeff4d2ff5596a72da862d024f9dbb1ff5bf67ca4f2921f77 sepolia-00046-4f4fe79f.era1
|
||||
9eb6dd3772ebfe63c37a22423310a5cb36c0d09defd66d7aa02a5b4c1283ca80 sepolia-00047-76b58fb3.era1
|
||||
82afedad4e56f86d836446f0c71616f8d88fdec11b93a6360dacc111cdb12db2 sepolia-00048-0fa9d93c.era1
|
||||
8a86584323e83a11499265f33ddabe0de1517b51c8e73985fa6eed15cfd63d0b sepolia-00049-d193ff47.era1
|
||||
b66621a148fb7acce05efe6aab89be5a965463a4c4c95a3350e64f3e8a684417 sepolia-00050-736b969f.era1
|
||||
49d47547284339de6f92e6495347dce0c45c5294f07c1b96fcfa510c43cfc206 sepolia-00051-466eb482.era1
|
||||
b075750d6efdde0cd755c90cd893b2917c56ab8674022fb36bb7a1305dc9bfed sepolia-00052-9752212c.era1
|
||||
36eb486d7dab70e759c72198b9f2505ac3fcb7f6a0ae2cca82fd9ade3d65e86d sepolia-00053-b2897233.era1
|
||||
9f024ea4e135fe6939f2fedb6a6821d61202139fa35378616fcf0274bdeac172 sepolia-00054-aedad6a8.era1
|
||||
2bc46d10add9c9903d554523f6ed0c8f5a632521c659d9f3f4af4c444221d08c sepolia-00055-19af5091.era1
|
||||
21900526ff5f0f31257bdabaae3fffa700eccd3cf3a6ba346b4bdf4347c97324 sepolia-00056-9ac921a1.era1
|
||||
c9a47d7db6af1b984e6ad8de2ea0eb0e73e442281d4fa497ba64c02caaf0b7d0 sepolia-00057-240f011a.era1
|
||||
10d38d3b6b66367a1c05922040073d6a62a2cbb843ea821dbd02f4d9f037c740 sepolia-00058-06d606d6.era1
|
||||
d391f44dfa9bab3ced35676cd964daeffe0f4e4028c3bd44fe41010b761c5229 sepolia-00059-8ea69a55.era1
|
||||
39a1bdf4d7c1dd8365f635c8da568b34d1c2e59aa5d7fe8745d6b7db2cf2c6ed sepolia-00060-d22079b0.era1
|
||||
cf41ae65b6391d8c44bb0128e66591a96d42fea0a7d35c30c8eca5bbe0790578 sepolia-00061-834d00e3.era1
|
||||
9bcff9fa47c2addc140857b23d6453583ca526c6cdf11144583db24acf33b1b2 sepolia-00062-d1230cc2.era1
|
||||
984c5662f6f36ece4e0ca9efc47dcb3e50e0aa581a874a680736f2f558cdd889 sepolia-00063-e1e9fa40.era1
|
||||
7582532b67144ea9499c28d935e7ac4bbf8f465c382a8a97ce04e56539b8c9af sepolia-00064-7f827781.era1
|
||||
945e3da724660df934484d68868d54fbfaa5feaacdd635f6185b56e88858ad8d sepolia-00065-9d8993d6.era1
|
||||
77fdff04d6d6233c4bab9811ad559a328fdb0447070b96e2f38ea7450639a00e sepolia-00066-8d516260.era1
|
||||
3486301cf48b1f727b46c0223c4a018e03d0857efedc2f616ea850b539139f93 sepolia-00067-7466141b.era1
|
||||
2954560d2b1c6ff48242312b7f5fe902c53dc93a3d1c7d9abf41633926ef72ac sepolia-00068-1bd11e8a.era1
|
||||
8f27de663991d7e4e8a7005dee7b94f991cfefad743507362eca5ad83f53e449 sepolia-00069-536ecd2c.era1
|
||||
2d0fa324482aab3db61718abfb2a5c084a55db5f39c4bce125255eec9a56a01a sepolia-00070-b27f7c5c.era1
|
||||
472b1c9ec83d9b6564a45d4363c856604d21b53d8c89a7b5810f2596c14f710a sepolia-00071-d2ef5349.era1
|
||||
3a0e6051e4c5f1a263b51f7a827d2276c7b25cb9d426f24f39fca33a7e76d623 sepolia-00072-b23578a9.era1
|
||||
734b46f0bb7150e1d705bff53a0cdedd3ed57ca1461f06deb517c8b74c4b25c9 sepolia-00073-8bacb416.era1
|
||||
28091f1a78dc398e2d47212087f652c8d35f07cda5457ee691b4191c3a6698c8 sepolia-00074-0e81003c.era1
|
||||
b743e3e1721e593562d9a144f49a927bbe2db70be9e4884521cae9500b0004cb sepolia-00075-09177034.era1
|
||||
6850f49599b95dcf1db54a3c5d126654ca21de4df27b7a24bc35c4d10fe6cdb7 sepolia-00076-0f2898a3.era1
|
||||
da57f20ff5a3944bd1fef22e7c914f2eda3e79dd7346b0e7c0a01e69cf538d7e sepolia-00077-61de3538.era1
|
||||
fc20551b69bacce502fa42ba5b0a223c9fb558d5f495948e38c060328f1ceb7e sepolia-00078-be407b86.era1
|
||||
ab72961502e1d8055886d744c077dbf60db8acba64c7879109b3ed573ce4cd02 sepolia-00079-d5c957be.era1
|
||||
216b65dba90adf1288d40fc689f6087ff2435bd379df482b130cd464cbd1d490 sepolia-00080-ae6ce310.era1
|
||||
e824e04c1cf15f142bdbb40161b64953767abd3ec7c40a726ce69d4ce646df08 sepolia-00081-daac26d6.era1
|
||||
d70d059609dda26e25af96d32b646c364e2ae44fe7b1cf9e2e8589d731e9f381 sepolia-00082-ecb8f0c2.era1
|
||||
a7dc282ca7fd7aa8d42693363aec655d6cfea330220233c7aad3f7955284d650 sepolia-00083-5b017cd4.era1
|
||||
2f1006528caa6d47729c4ef0a699c441cb5d2c201afb6563ab51184429294020 sepolia-00084-8c1f92cd.era1
|
||||
374494754cfd728b1b4e4a00db810012346ba8c4dc9cf2ad1d80187999ab868b sepolia-00085-9180667c.era1
|
||||
7ff9b360a0656ddbace4032d249f5cf06068ee3f8be526d2a4ae0ed6eaef046f sepolia-00086-1aca7add.era1
|
||||
de592f4bde8de973d8c82a7d89dbe21cf75bb0b28204567ea4f5012b8be1f28c sepolia-00087-0e100944.era1
|
||||
fb6eb1e1e1ce97e03b39fe2d53e3595d759701a65cc861d0568c04ee7e2ec03c sepolia-00088-fcdd7aa5.era1
|
||||
4d178ce50f5436f5ba2347d87dc79e9a2e18c17d3056eb77c7c8dbb9b03bb986 sepolia-00089-b23b2368.era1
|
||||
21e919f0d32137e7924d54ddcf1e697bb5b57bbb7cc12a169475df993c4c1d9f sepolia-00090-b78edeb7.era1
|
||||
cb7ae025ca17c5f52ef7910147a47a0d3680c26f9d2d109d6af330f93f95b735 sepolia-00091-4c81cdca.era1
|
||||
1e0249554f150b63cb7dd57c53c624c197b415401c0a1f3986c2f06e8aaa795c sepolia-00092-ddbf3a4c.era1
|
||||
8d950d8c13a4e002514a47e6d6ebcad4bf7e5808398641a5c04863d9c90f0f87 sepolia-00093-a52a45ec.era1
|
||||
b8720324f04dca4e0046b12ec3b8a2b862158f3156c4d1fc2947340b1fc1a9cb sepolia-00094-839ea4e5.era1
|
||||
8c4134bbb21d36fbd7ba608eb1e485e76cb0fd56c4aa8032be1b6848757856d9 sepolia-00095-bccc0958.era1
|
||||
e3ad213e760ea2320ff52caf5de32ced37d6b7ad0f99384785ca42a7e58d1cd9 sepolia-00096-429b2f39.era1
|
||||
9c324e109f77ebc1750dd2a0941904ff820047def91f8b832e61c01b31297f38 sepolia-00097-6f74184d.era1
|
||||
26efc4469736b283fd80d9105fc85a70dbec209fbb0e1e40a905a9406e1bb3cc sepolia-00098-40286059.era1
|
||||
e14134b3aad0533366f9aeb19cf2561cf1e81b989a0b83f12b05493948dd71ea sepolia-00099-dd1ef4f4.era1
|
||||
7283a437244545e784d51186c0d7d4809808dd472850ca3cd2d7a40f4883b50d sepolia-00100-bd3d6bf4.era1
|
||||
b67270e94377703aac798d8323c0a91b451d36806775013a2b8b12db04797851 sepolia-00101-41676e0d.era1
|
||||
dd6ea805242a67c6255867f64fc5fbc941a98318b5ea1d5e2d42d42c6910493c sepolia-00102-4f3c1fe1.era1
|
||||
d6f6da94b16a201b304936cb5f798b6fedba76ead056cd2c4dc860c21d82b8dd sepolia-00103-e13a95f0.era1
|
||||
e2309561f7a86eddfe182dc86fb38e5454c5a272cc77b3d0d7c6ce987ddc874c sepolia-00104-4e49b068.era1
|
||||
deff905084758be35bf0b8cb640b1b283232abbc6153388a4275c767ea0bdf0e sepolia-00105-4ab7d9ea.era1
|
||||
964f9679a39fe06192a7025f14404cb1f6542071665f1c2f739f62fdb55e9274 sepolia-00106-5ec678af.era1
|
||||
8fd75370992f97e77a2cd3663bc1fb9b2bcec9b64a0439de7846d7c6321e6d93 sepolia-00107-44721c55.era1
|
||||
7e8219aa13d6a7b32c3a69888a2c5f446703eada3fdcde841ba26edfe3cb4453 sepolia-00108-3b4ad768.era1
|
||||
24e8d2f8d01abf63d97365bafa914dd5203f4e807bc6655dc0673b9f87215ebd sepolia-00109-16e54758.era1
|
||||
edadb0ebec6c4b7b54237e5802cc59fd962e23cf690ff10ab52c10258d7b7827 sepolia-00110-b5da103b.era1
|
||||
4faebfbdc009188cc8cf0ffe4aeca9c93ac5b3abe763cfb165d8bed37b12b51c sepolia-00111-15c53aba.era1
|
||||
40210479bd14ca3f8db76b288c2ed4e8f3d5b5ce8116ba8d8fe3133b0cc765ad sepolia-00112-8e2144e3.era1
|
||||
92ed406d564ed1504bc7ff91298a419fe72a710585256b1830040ba3d1adbde8 sepolia-00113-1e1aac0a.era1
|
||||
7e771bba1aa9c87be155a9a7d62d3479240f21934a3d763cafa5f3cd4ce49f8a sepolia-00114-0d7a4b23.era1
|
||||
51d71ed3674f5d70f719b3251e29a0825f2573b1bfd1a92940b013806aaf4fd3 sepolia-00115-8e983a59.era1
|
||||
6abea02d82f7a30b67d6de4516bbc711855eea3d891ea42fa0124c101f981f86 sepolia-00116-1708ba6e.era1
|
||||
88e7222e35b1275483b6eec11bf8d4417602fdb16b611d6c154abd8d0ec0b49a sepolia-00117-86962046.era1
|
||||
6ddeae324cc17337fa23c71b242ab1675b72f43fb393af7e971edec20451f864 sepolia-00118-934e9f5f.era1
|
||||
914c35142f7244d07b670b1f990c8bb5997119db451e8dcd8a67c8d71fcc6af2 sepolia-00119-4d88db3e.era1
|
||||
eedec437c182f330f2f5f3c530cce3bc59f0c99c9e6cd36288a588adba7679bb sepolia-00120-1a3274d5.era1
|
||||
4914f6a8cf63cb806c5736431f6402d9a99c2de953da05a17069fa15cf8c695b sepolia-00121-62b79757.era1
|
||||
b9a0138c77dcbb2b4784813204ec8bc30407cd8daf3501707719985acff4a1cb sepolia-00122-41d14652.era1
|
||||
57453e0eaa2bf46a13c94d9e96ee98943a46012c2bdd9da5d7c77064e803fbfd sepolia-00123-67f86708.era1
|
||||
1e927fdb13cebda45a963c8cec921b16c795fb3020cad9054fa1fb7f4cb528cd sepolia-00124-488cb226.era1
|
||||
8015846c8c3663005a8a6d59347fb672a602e6bbff028989355a01f5f6d2b183 sepolia-00125-8a21f7da.era1
|
||||
8807f6abe63b76f31d99cf14404fee8bb5eba3a814e2d08b9d85540ef9742550 sepolia-00126-6569478a.era1
|
||||
28dc46c8120426b395cf9b5ff781962a3212766b24511b2dc3d2172568b10021 sepolia-00127-1312ab23.era1
|
||||
ed66119ec9dae6c61952cbf40b9dc5770cd80e659a83dd18e1a5f2ffa947a04b sepolia-00128-1c95bdc6.era1
|
||||
a4a68dd54b77f21939ddb1ec9ef9ae0a3551d6d53b8f9578b01ad8e32e5c8d8e sepolia-00129-f32fdefa.era1
|
||||
e54d6d4b95a985f37391750b3bf3bc0f9f4e2f790805797ee198d77287433ca2 sepolia-00130-8f31c590.era1
|
||||
577fe68c5c33e47af8823ac573709ed9d63df0d7f9aec02a7907e3e6e5922d67 sepolia-00131-5aaad354.era1
|
||||
c3beada9620c55b6d0dbdab8f66b735da0b3d62d5c5eb2c3b5369619549fb3e0 sepolia-00132-2147d970.era1
|
||||
12843e128e76d5636f857c8c6801e433757fc4f8aba159bc2d4e6c41f2c94abd sepolia-00133-690cc16f.era1
|
||||
e5e4904ec83ebcebccc1aae1de89622a50bf6f735d985234da88fa1ced1787c0 sepolia-00134-fe276e6b.era1
|
||||
9c618458f6d76a09f16c86baf526dc7f2310ce92d862f73392144987c04b39ed sepolia-00135-ce40efd0.era1
|
||||
1d032d1279c3841c65385f85463226df04a98a3ccc5d7bf461f042492230615c sepolia-00136-adc6e1bb.era1
|
||||
a3aa401ca2bc8cb8b104667c6f68e28d00e37813a1a993d8abd1ec5c3555f71d sepolia-00137-805a333a.era1
|
||||
105c1e50e73715bf58d42f6bbe49bff36dc0051bfb477efc77ebf14b5ba98974 sepolia-00138-65ff170f.era1
|
||||
ad53fc0e4d2718ce105fa3a7ce5574dbb945f49b2fa7470a406179c45738fc46 sepolia-00139-1c86d86a.era1
|
||||
c27ae427ca0052686a3ac8a3e568d1dae30449f9edba465f218c147074d61ccc sepolia-00140-e7f36280.era1
|
||||
143d09ffb26cd2a1bb334958d92b77a92a8f9ac99c01059d22bee4e4d8bc3212 sepolia-00141-5d3d0d5a.era1
|
||||
378a77f0b0c4072297de4c382cece46be521a6c6935db8eb499dd99857d566aa sepolia-00142-e45aa418.era1
|
||||
35855d5bd75dff01248dd5b2080babf6127d87c157a987124f7a258cfb9a5ecb sepolia-00143-70379f59.era1
|
||||
58f07f98d8e9486acd01ec212518b1c0d7764e68263f9b1e9c51cf5e597a0b6b sepolia-00144-c345e75c.era1
|
||||
17caa603e2d1f9c01c9117754f0bb4c8ce42c58da806f758f99468d9f8c1c47b sepolia-00145-9f4bb4c5.era1
|
||||
d05a92b8fe1306397b98a6202b18f66c9bc259bd41c25b3d8211c09813e3a9ac sepolia-00146-3336d26b.era1
|
||||
b7120a656d06d4010a06a2abadd809ee7f26707cf313e9de5cb1bd172745ad23 sepolia-00147-6b007873.era1
|
||||
aa45cc5747e9c21ccfc44c87f98717702551a0511f423b2624a34af58ba798fc sepolia-00148-35f8a5b4.era1
|
||||
952843d531ea4d9ee9c456e50e77c50f75a559bff13326dd593271f90cd55129 sepolia-00149-01b6e9ca.era1
|
||||
2d93d1d84e0f4013d6daab61d66907488223a2f1b50e4a22d3ba1da24a9c6a84 sepolia-00150-0c372e80.era1
|
||||
6b516e3f31720c54453c5ce9bbac68300615f455a7cf4b1dc6592a5d5432ee89 sepolia-00151-c15a5955.era1
|
||||
82023cab017453e8fcf49ace9befbaef4af167e0e6c6976456bf251f803b5d51 sepolia-00152-ef7ac893.era1
|
||||
6ace677a17380127349ba0710fd02ff38ba185526cc2d9bbb6f05375d6f163f6 sepolia-00153-0e4073c2.era1
|
||||
66db5e0786cb9963ed66a0e31773b30d3405621a7636c62ffbb0b84ff41705bd sepolia-00154-298c3549.era1
|
||||
cf797283087a079773770c1bebfecb113a4155e0d77d5713c702478e2c24f64a sepolia-00155-a99309bb.era1
|
||||
b143cc994d0b544cd08b2a1f66308718ea49f3f68adef8e62300e91f8048130a sepolia-00156-8c33148f.era1
|
||||
cbc20f9f767324438a237c3114a1c4e65dda74d4a655d55a7cc3e63e62f373f8 sepolia-00157-ab02fc10.era1
|
||||
c8368d1752e66bcd7cabc653477f9aec0071027f91112a8cf71a945db23404e7 sepolia-00158-17992856.era1
|
||||
03a1a66eab3a7cbc63b29cfbe2e70285940e64db4af6d23ab70592c820f8d3ea sepolia-00159-b3448cf6.era1
|
||||
bf9dcd8400e56c58cd3280f0a3d7aab23a7eaf594e67db5387f40101ad8e8c92 sepolia-00160-acad7054.era1
|
||||
03a3b975b2a78a4b28afd9ee089440465ceefd32bd1d82b74ae69caa80b6fb6c sepolia-00161-36e611bd.era1
|
||||
ddf201b59ebc77eccdec97b0a5a7d7a89dda02928ff7c5cca8868b5aba84f533 sepolia-00162-3557cc58.era1
|
||||
041be469daf6f7abaa0cfe66c4d68b7d9d223383553c16f078edb3a1fa3d0d92 sepolia-00163-638b9afe.era1
|
||||
68d151048412318cbd0af98efa27d7ee16e2fc58830d93b4d76b66cb632ebfc3 sepolia-00164-fbc67b64.era1
|
||||
3025771380736a5de790ead991c0512a4c73106fd1d2f169863d395964238ac1 sepolia-00165-442b83a1.era1
|
||||
6d6b1273a6fc353c6999b4c9be301660a404b78cced911e230991305da990e3c sepolia-00166-018915fc.era1
|
||||
01052c4f779e4ec276498d4192b8ad11ad6d2d3992224bb62ea882a80deda935 sepolia-00167-0df2b4c5.era1
|
||||
2f8f3034eb7e41c5add0e639e21b52c0794f36bb5918274849402c5fe0d247e6 sepolia-00168-7cab66f5.era1
|
||||
504b0203d995e5ef6aee679b048c276f45b05dc68a6fd319a0f4f0133b230a77 sepolia-00169-75850131.era1
|
||||
b277df79fe5a83189f9619b275a9aaab63db8506539c56997be61fd5a3552bb9 sepolia-00170-c2044b78.era1
|
||||
1af0a1778fc5d2a53aaca06dbda6671b91ae3f56da73a6c6708bc729e5fb9efd sepolia-00171-af44ff2e.era1
|
||||
a977654c0ed243ccce7b5514914ec94d70a3a742ff014ac52b0654e21940f2da sepolia-00172-bd76a8d9.era1
|
||||
146672b2e63c320c9474ecf41d7f32f31f5298512cc8f911074c88789849d935 sepolia-00173-b6924da5.era1
|
||||
82af1c5bc51f9e9ae1be5d0e0827ea755d104ed373b374c08ef401115491671f sepolia-00174-2a61fac5.era1
|
||||
627ee6cf9282191ff32d291fc0437b812bad01f158b5ca42ae717b7bbb125f8c sepolia-00175-1cc76405.era1
|
||||
6ee38f117646a47f6ab2f666fa6460bda770c194ae4a29744534e9bfd9e7ba3b sepolia-00176-20daa2c6.era1
|
||||
28e7e49e71dd8581cc484d8be89fb083a79577b06f26c1e3bfd4a57bdf24c5e7 sepolia-00177-379df7e4.era1
|
||||
055c365cfeda3495a5e2b334803dfffe7af2a584745acabc42b5ad146b250534 sepolia-00178-88c9d2aa.era1
|
||||
bca9a920232983636b7cf20fc8319f891df94ba0860f51b5a052290b58d9e3ae sepolia-00179-8329b80c.era1
|
||||
e894a6f3ce0ede2825aaafa64122a5f458cd3b2ecaeae20a44a59fd668971bb0 sepolia-00180-22de0418.era1
|
||||
83bfe0d7c49c3a108a4ed044119a3549f2f4b0f8e74c971150b6977b7d46a250 sepolia-00181-b3fbe6f2.era1
|
||||
3b806177536ac3615f376440c0589696ea746eea0215bb3b175df8f4f1e73894 sepolia-00182-a4f0a8a1.era1
|
||||
115
internal/era/eradl/eradl.go
Normal file
115
internal/era/eradl/eradl.go
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package eradl implements downloading of era1 files.
|
||||
package eradl
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
|
||||
"github.com/ethereum/go-ethereum/internal/download"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
)
|
||||
|
||||
//go:embed checksums_mainnet.txt
|
||||
var mainnetDB []byte
|
||||
|
||||
//go:embed checksums_sepolia.txt
|
||||
var sepoliaDB []byte
|
||||
|
||||
type Loader struct {
|
||||
csdb *download.ChecksumDB
|
||||
network string
|
||||
baseURL *url.URL
|
||||
}
|
||||
|
||||
// New creates an era1 loader for the given server URL and network name.
|
||||
func New(baseURL string, network string) (*Loader, error) {
|
||||
var checksums []byte
|
||||
switch network {
|
||||
case "mainnet":
|
||||
checksums = mainnetDB
|
||||
case "sepolia":
|
||||
checksums = sepoliaDB
|
||||
default:
|
||||
return nil, fmt.Errorf("missing era1 checksum definitions for network %q", network)
|
||||
}
|
||||
|
||||
csdb, err := download.ParseChecksums(checksums)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid checksums: %v", err)
|
||||
}
|
||||
|
||||
base, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid base URL %q: %v", baseURL, err)
|
||||
}
|
||||
if base.Scheme != "http" && base.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid base URL scheme, expected http(s): %q", baseURL)
|
||||
}
|
||||
|
||||
l := &Loader{
|
||||
network: network,
|
||||
csdb: csdb,
|
||||
baseURL: base,
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// DownloadAll downloads all known era1 files to the given directory.
|
||||
func (l *Loader) DownloadAll(destDir string) error {
|
||||
for file := range l.csdb.Files() {
|
||||
if err := l.download(file, destDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadBlockRange fetches the era1 files for the given block range.
|
||||
func (l *Loader) DownloadBlockRange(start, end uint64, destDir string) error {
|
||||
startEpoch := start / uint64(era.MaxEra1Size)
|
||||
endEpoch := end / uint64(era.MaxEra1Size)
|
||||
return l.DownloadEpochRange(startEpoch, endEpoch, destDir)
|
||||
}
|
||||
|
||||
// DownloadEpochRange fetches the era1 files in the given epoch range.
|
||||
func (l *Loader) DownloadEpochRange(start, end uint64, destDir string) error {
|
||||
pat := regexp.MustCompile(regexp.QuoteMeta(l.network) + "-([0-9]+)-[0-9a-f]+\\.era1")
|
||||
for file := range l.csdb.Files() {
|
||||
m := pat.FindStringSubmatch(file)
|
||||
if len(m) == 2 {
|
||||
fileEpoch, _ := strconv.Atoi(m[1])
|
||||
if uint64(fileEpoch) >= start && uint64(fileEpoch) <= end {
|
||||
if err := l.download(file, destDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Loader) download(file, destDir string) error {
|
||||
url := l.baseURL.JoinPath(file).String()
|
||||
dest := filepath.Join(destDir, file)
|
||||
return l.csdb.DownloadFile(url, dest)
|
||||
}
|
||||
Loading…
Reference in a new issue