diff --git a/build/ci.go b/build/ci.go
index b2ff829d3f..6e6d58cbe9 100644
--- a/build/ci.go
+++ b/build/ci.go
@@ -59,6 +59,7 @@ import (
"github.com/cespare/cp"
"github.com/ethereum/go-ethereum/crypto/signify"
"github.com/ethereum/go-ethereum/internal/build"
+ "github.com/ethereum/go-ethereum/internal/download"
"github.com/ethereum/go-ethereum/internal/version"
)
@@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
// Configure the toolchain.
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
if *dlgo {
- csdb := build.MustLoadChecksums("build/checksums.txt")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
tc.Root = build.DownloadGo(csdb)
}
// Disable CLI markdown doc generation in release builds.
@@ -285,7 +286,7 @@ func doTest(cmdline []string) {
flag.CommandLine.Parse(cmdline)
// Get test fixtures.
- csdb := build.MustLoadChecksums("build/checksums.txt")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
downloadSpecTestFixtures(csdb, *cachedir)
// Configure the toolchain.
@@ -329,8 +330,8 @@ func doTest(cmdline []string) {
}
// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
-func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
- executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
+func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
+ executionSpecTestsVersion, err := csdb.FindVersion("spec-tests")
if err != nil {
log.Fatal(err)
}
@@ -444,8 +445,8 @@ func doLint(cmdline []string) {
// downloadLinter downloads and unpacks golangci-lint.
func downloadLinter(cachedir string) string {
- csdb := build.MustLoadChecksums("build/checksums.txt")
- version, err := build.Version(csdb, "golangci")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
+ version, err := csdb.FindVersion("golangci")
if err != nil {
log.Fatal(err)
}
@@ -497,8 +498,8 @@ func protocArchiveBaseName() (string, error) {
// in the generate command. It returns the full path of the directory
// containing the 'protoc-gen-go' executable.
func downloadProtocGenGo(cachedir string) string {
- csdb := build.MustLoadChecksums("build/checksums.txt")
- version, err := build.Version(csdb, "protoc-gen-go")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
+ version, err := csdb.FindVersion("protoc-gen-go")
if err != nil {
log.Fatal(err)
}
@@ -531,8 +532,8 @@ func downloadProtocGenGo(cachedir string) string {
// files as a CI step. It returns the full path to the directory containing
// the protoc executable.
func downloadProtoc(cachedir string) string {
- csdb := build.MustLoadChecksums("build/checksums.txt")
- version, err := build.Version(csdb, "protoc")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
+ version, err := csdb.FindVersion("protoc")
if err != nil {
log.Fatal(err)
}
@@ -826,11 +827,11 @@ func doDebianSource(cmdline []string) {
// downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
// to bootstrap the builder Go.
func downloadGoBootstrapSources(cachedir string) []string {
- csdb := build.MustLoadChecksums("build/checksums.txt")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
var bundles []string
for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
- gobootVersion, err := build.Version(csdb, booter)
+ gobootVersion, err := csdb.FindVersion(booter)
if err != nil {
log.Fatal(err)
}
@@ -847,8 +848,8 @@ func downloadGoBootstrapSources(cachedir string) []string {
// downloadGoSources downloads the Go source tarball.
func downloadGoSources(cachedir string) string {
- csdb := build.MustLoadChecksums("build/checksums.txt")
- dlgoVersion, err := build.Version(csdb, "golang")
+ csdb := download.MustLoadChecksums("build/checksums.txt")
+ dlgoVersion, err := csdb.FindVersion("golang")
if err != nil {
log.Fatal(err)
}
@@ -1181,5 +1182,6 @@ func doPurge(cmdline []string) {
}
func doSanityCheck() {
- build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
+ csdb := download.MustLoadChecksums("build/checksums.txt")
+ csdb.VerifyAll()
}
diff --git a/internal/build/download.go b/internal/build/download.go
deleted file mode 100644
index 50268227a5..0000000000
--- a/internal/build/download.go
+++ /dev/null
@@ -1,151 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see .
-
-package build
-
-import (
- "bufio"
- "crypto/sha256"
- "encoding/hex"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "strings"
-)
-
-// ChecksumDB keeps file checksums.
-type ChecksumDB struct {
- allChecksums []string
-}
-
-// MustLoadChecksums loads a file containing checksums.
-func MustLoadChecksums(file string) *ChecksumDB {
- content, err := os.ReadFile(file)
- if err != nil {
- log.Fatal("can't load checksum file: " + err.Error())
- }
- return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
-}
-
-// Verify checks whether the given file is valid according to the checksum database.
-func (db *ChecksumDB) Verify(path string) error {
- fd, err := os.Open(path)
- if err != nil {
- return err
- }
- defer fd.Close()
-
- h := sha256.New()
- if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
- return err
- }
- fileHash := hex.EncodeToString(h.Sum(nil))
- if !db.findHash(filepath.Base(path), fileHash) {
- return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
- }
- return nil
-}
-
-func (db *ChecksumDB) findHash(basename, hash string) bool {
- want := hash + " " + basename
- for _, line := range db.allChecksums {
- if strings.TrimSpace(line) == want {
- return true
- }
- }
- return false
-}
-
-// DownloadFile downloads a file and verifies its checksum.
-func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
- if err := db.Verify(dstPath); err == nil {
- fmt.Printf("%s is up-to-date\n", dstPath)
- return nil
- }
- fmt.Printf("%s is stale\n", dstPath)
- fmt.Printf("downloading from %s\n", url)
-
- resp, err := http.Get(url)
- if err != nil {
- return fmt.Errorf("download error: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("download error: status %d", resp.StatusCode)
- }
- if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
- return err
- }
- fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
- if err != nil {
- return err
- }
- dst := newDownloadWriter(fd, resp.ContentLength)
- _, err = io.Copy(dst, resp.Body)
- dst.Close()
- if err != nil {
- return err
- }
- return db.Verify(dstPath)
-}
-
-type downloadWriter struct {
- file *os.File
- dstBuf *bufio.Writer
- size int64
- written int64
- lastpct int64
-}
-
-func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
- return &downloadWriter{
- file: dst,
- dstBuf: bufio.NewWriter(dst),
- size: size,
- }
-}
-
-func (w *downloadWriter) Write(buf []byte) (int, error) {
- n, err := w.dstBuf.Write(buf)
-
- // Report progress.
- w.written += int64(n)
- pct := w.written * 10 / w.size * 10
- if pct != w.lastpct {
- if w.lastpct != 0 {
- fmt.Print("...")
- }
- fmt.Print(pct, "%")
- w.lastpct = pct
- }
- return n, err
-}
-
-func (w *downloadWriter) Close() error {
- if w.lastpct > 0 {
- fmt.Println() // Finish the progress line.
- }
- flushErr := w.dstBuf.Flush()
- closeErr := w.file.Close()
- if flushErr != nil {
- return flushErr
- }
- return closeErr
-}
diff --git a/internal/build/gotool.go b/internal/build/gotool.go
index 2a47460418..172fa13464 100644
--- a/internal/build/gotool.go
+++ b/internal/build/gotool.go
@@ -24,6 +24,8 @@ import (
"path/filepath"
"runtime"
"strings"
+
+ "github.com/ethereum/go-ethereum/internal/download"
)
type GoToolchain struct {
@@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
// DownloadGo downloads the Go binary distribution and unpacks it into a temporary
// directory. It returns the GOROOT of the unpacked toolchain.
-func DownloadGo(csdb *ChecksumDB) string {
- version, err := Version(csdb, "golang")
+func DownloadGo(csdb *download.ChecksumDB) string {
+ version, err := csdb.FindVersion("golang")
if err != nil {
log.Fatal(err)
}
@@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
}
return goroot
}
-
-// Version returns the versions defined in the checksumdb.
-func Version(csdb *ChecksumDB, version string) (string, error) {
- for _, l := range csdb.allChecksums {
- if !strings.HasPrefix(l, "# version:") {
- continue
- }
- v := strings.Split(l, ":")[1]
- parts := strings.Split(v, " ")
- if len(parts) != 2 {
- log.Print("Erroneous version-string", "v", l)
- continue
- }
- if parts[0] == version {
- return parts[1], nil
- }
- }
- return "", fmt.Errorf("no version found for '%v'", version)
-}
-
-// DownloadAndVerifyChecksums downloads all files and checks that they match
-// the checksum given in checksums.txt.
-// This task can be used to sanity-check new checksums.
-func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
- var (
- base = ""
- ucache = os.TempDir()
- )
- for _, l := range csdb.allChecksums {
- if strings.HasPrefix(l, "# https://") {
- base = l[2:]
- continue
- }
- if strings.HasPrefix(l, "#") {
- continue
- }
- hashFile := strings.Split(l, " ")
- if len(hashFile) != 2 {
- continue
- }
- file := hashFile[1]
- url := base + file
- dst := filepath.Join(ucache, file)
- if err := csdb.DownloadFile(url, dst); err != nil {
- log.Print(err)
- }
- }
-}
diff --git a/internal/download/download.go b/internal/download/download.go
new file mode 100644
index 0000000000..f388350607
--- /dev/null
+++ b/internal/download/download.go
@@ -0,0 +1,261 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+// Package download implements checksum-verified file downloads.
+package download
+
+import (
+ "bufio"
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+)
+
+// ChecksumDB keeps file checksums and tool versions.
+type ChecksumDB struct {
+ hashes []hashEntry
+ versions []versionEntry
+}
+
+type versionEntry struct {
+ name string
+ version string
+}
+
+type urlEntry struct {
+ url string
+}
+
+type hashEntry struct {
+ hash string
+ file string
+ u *urlEntry
+}
+
+// MustLoadChecksums loads a file containing checksums.
+func MustLoadChecksums(file string) *ChecksumDB {
+ content, err := os.ReadFile(file)
+ if err != nil {
+ log.Fatal("can't load checksum file: " + err.Error())
+ }
+ db, err := ParseChecksums(content)
+ if err != nil {
+ log.Fatalf("invalid checksums in %s: %v", file, err)
+ }
+ return db
+}
+
+// ParseChecksums parses a checksum database.
+func ParseChecksums(input []byte) (*ChecksumDB, error) {
+ var (
+ csdb = new(ChecksumDB)
+ rd = bytes.NewBuffer(input)
+ lastURL *urlEntry
+ )
+ for lineNum := 1; ; lineNum++ {
+ line, err := rd.ReadString('\n')
+ if err == io.EOF {
+ break
+ }
+ line = strings.TrimSpace(line)
+ switch {
+ case line == "":
+ // Blank lines are allowed, and they reset the current urlEntry.
+ lastURL = nil
+
+ case strings.HasPrefix(line, "#"):
+ // It's a comment. Some comments have special meaning.
+ content := strings.TrimLeft(line, "# ")
+ switch {
+ case strings.HasPrefix(content, "version:"):
+ // Version comments define the version of a tool.
+ v := strings.Split(content, ":")[1]
+ parts := strings.Split(v, " ")
+ if len(parts) != 2 {
+ return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
+ }
+ csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
+
+ case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
+ // URL comments define the URL where the following files are found. Here
+ // we keep track of the last found urlEntry and attach it to each file later.
+ lastURL = &urlEntry{content}
+ }
+
+ default:
+ // It's a file hash entry.
+ fields := strings.Fields(line)
+ if len(fields) != 2 {
+ return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
+ }
+ csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
+ }
+ }
+ return csdb, nil
+}
+
+// Verify checks whether the given file is valid according to the checksum database.
+func (db *ChecksumDB) Verify(path string) error {
+ fd, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+ defer fd.Close()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
+ return err
+ }
+ fileHash := hex.EncodeToString(h.Sum(nil))
+ if !db.containsHash(filepath.Base(path), fileHash) {
+ return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
+ }
+ return nil
+}
+
+// DownloadFile downloads a file and verifies its checksum.
+func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
+ if err := db.Verify(dstPath); err == nil {
+ fmt.Printf("%s is up-to-date\n", dstPath)
+ return nil
+ }
+ fmt.Printf("%s is stale\n", dstPath)
+ fmt.Printf("downloading from %s\n", url)
+
+ resp, err := http.Get(url)
+ if err != nil {
+ return fmt.Errorf("download error: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("download error: status %d", resp.StatusCode)
+ }
+ if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+ return err
+ }
+ fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ dst := newDownloadWriter(fd, resp.ContentLength)
+ _, err = io.Copy(dst, resp.Body)
+ dst.Close()
+ if err != nil {
+ return err
+ }
+ return db.Verify(dstPath)
+}
+
+// FindVersion returns the current known version of a tool, if it is defined in the file.
+func (db *ChecksumDB) FindVersion(tool string) (string, error) {
+ for _, e := range db.versions {
+ if e.name == tool {
+ return e.version, nil
+ }
+ }
+ return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
+}
+
+// FindURL gets the URL for a file.
+func (db *ChecksumDB) FindURL(basename string) (string, error) {
+ for _, e := range db.hashes {
+ if e.file == basename {
+ if e.u == nil {
+ return "", fmt.Errorf("file %q has no URL defined", e.file)
+ }
+ return e.u.url + e.file, nil
+ }
+ }
+ return "", fmt.Errorf("file %q does not exist in checksum database", basename)
+}
+
+// VerifyAll downloads all files and checks that they match the checksum given in
+// the database. This task can be used to sanity-check new checksums.
+func (db *ChecksumDB) VerifyAll() {
+ var (
+ tmp = os.TempDir()
+ )
+ for _, e := range db.hashes {
+ if e.u == nil {
+ log.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
+ continue
+ }
+ url := e.u.url + e.file
+ dst := filepath.Join(tmp, e.file)
+ if err := db.DownloadFile(url, dst); err != nil {
+ log.Print(err)
+ }
+ }
+}
+
+func (db *ChecksumDB) containsHash(basename, hash string) bool {
+ return slices.ContainsFunc(db.hashes, func(e hashEntry) bool {
+ return e.file == basename && e.hash == hash
+ })
+}
+
+type downloadWriter struct {
+ file *os.File
+ dstBuf *bufio.Writer
+ size int64
+ written int64
+ lastpct int64
+}
+
+func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
+ return &downloadWriter{
+ file: dst,
+ dstBuf: bufio.NewWriter(dst),
+ size: size,
+ }
+}
+
+func (w *downloadWriter) Write(buf []byte) (int, error) {
+ n, err := w.dstBuf.Write(buf)
+
+ // Report progress.
+ w.written += int64(n)
+ pct := w.written * 10 / w.size * 10
+ if pct != w.lastpct {
+ if w.lastpct != 0 {
+ fmt.Print("...")
+ }
+ fmt.Print(pct, "%")
+ w.lastpct = pct
+ }
+ return n, err
+}
+
+func (w *downloadWriter) Close() error {
+ if w.lastpct > 0 {
+ fmt.Println() // Finish the progress line.
+ }
+ flushErr := w.dstBuf.Flush()
+ closeErr := w.file.Close()
+ if flushErr != nil {
+ return flushErr
+ }
+ return closeErr
+}