From 13d72092b032a59450026c04f128250ed6295fcc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 13 May 2025 23:15:07 +0200 Subject: [PATCH] internal/build: refactor file downloader into separate package --- build/ci.go | 32 +++-- internal/build/download.go | 151 -------------------- internal/build/gotool.go | 54 +------ internal/download/download.go | 261 ++++++++++++++++++++++++++++++++++ 4 files changed, 282 insertions(+), 216 deletions(-) delete mode 100644 internal/build/download.go create mode 100644 internal/download/download.go diff --git a/build/ci.go b/build/ci.go index b2ff829d3f..6e6d58cbe9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -59,6 +59,7 @@ import ( "github.com/cespare/cp" "github.com/ethereum/go-ethereum/crypto/signify" "github.com/ethereum/go-ethereum/internal/build" + "github.com/ethereum/go-ethereum/internal/download" "github.com/ethereum/go-ethereum/internal/version" ) @@ -190,7 +191,7 @@ func doInstall(cmdline []string) { // Configure the toolchain. tc := build.GoToolchain{GOARCH: *arch, CC: *cc} if *dlgo { - csdb := build.MustLoadChecksums("build/checksums.txt") + csdb := download.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb) } // Disable CLI markdown doc generation in release builds. @@ -285,7 +286,7 @@ func doTest(cmdline []string) { flag.CommandLine.Parse(cmdline) // Get test fixtures. - csdb := build.MustLoadChecksums("build/checksums.txt") + csdb := download.MustLoadChecksums("build/checksums.txt") downloadSpecTestFixtures(csdb, *cachedir) // Configure the toolchain. @@ -329,8 +330,8 @@ func doTest(cmdline []string) { } // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures. -func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { - executionSpecTestsVersion, err := build.Version(csdb, "spec-tests") +func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string { + executionSpecTestsVersion, err := csdb.FindVersion("spec-tests") if err != nil { log.Fatal(err) } @@ -444,8 +445,8 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "golangci") + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("golangci") if err != nil { log.Fatal(err) } @@ -497,8 +498,8 @@ func protocArchiveBaseName() (string, error) { // in the generate command. It returns the full path of the directory // containing the 'protoc-gen-go' executable. func downloadProtocGenGo(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "protoc-gen-go") + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("protoc-gen-go") if err != nil { log.Fatal(err) } @@ -531,8 +532,8 @@ func downloadProtocGenGo(cachedir string) string { // files as a CI step. It returns the full path to the directory containing // the protoc executable. func downloadProtoc(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "protoc") + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("protoc") if err != nil { log.Fatal(err) } @@ -826,11 +827,11 @@ func doDebianSource(cmdline []string) { // downloadGoBootstrapSources downloads the Go source tarball(s) that will be used // to bootstrap the builder Go. func downloadGoBootstrapSources(cachedir string) []string { - csdb := build.MustLoadChecksums("build/checksums.txt") + csdb := download.MustLoadChecksums("build/checksums.txt") var bundles []string for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} { - gobootVersion, err := build.Version(csdb, booter) + gobootVersion, err := csdb.FindVersion(booter) if err != nil { log.Fatal(err) } @@ -847,8 +848,8 @@ func downloadGoBootstrapSources(cachedir string) []string { // downloadGoSources downloads the Go source tarball. func downloadGoSources(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - dlgoVersion, err := build.Version(csdb, "golang") + csdb := download.MustLoadChecksums("build/checksums.txt") + dlgoVersion, err := csdb.FindVersion("golang") if err != nil { log.Fatal(err) } @@ -1181,5 +1182,6 @@ func doPurge(cmdline []string) { } func doSanityCheck() { - build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt")) + csdb := download.MustLoadChecksums("build/checksums.txt") + csdb.VerifyAll() } diff --git a/internal/build/download.go b/internal/build/download.go deleted file mode 100644 index 50268227a5..0000000000 --- a/internal/build/download.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "bufio" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "strings" -) - -// ChecksumDB keeps file checksums. -type ChecksumDB struct { - allChecksums []string -} - -// MustLoadChecksums loads a file containing checksums. -func MustLoadChecksums(file string) *ChecksumDB { - content, err := os.ReadFile(file) - if err != nil { - log.Fatal("can't load checksum file: " + err.Error()) - } - return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")} -} - -// Verify checks whether the given file is valid according to the checksum database. -func (db *ChecksumDB) Verify(path string) error { - fd, err := os.Open(path) - if err != nil { - return err - } - defer fd.Close() - - h := sha256.New() - if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { - return err - } - fileHash := hex.EncodeToString(h.Sum(nil)) - if !db.findHash(filepath.Base(path), fileHash) { - return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path)) - } - return nil -} - -func (db *ChecksumDB) findHash(basename, hash string) bool { - want := hash + " " + basename - for _, line := range db.allChecksums { - if strings.TrimSpace(line) == want { - return true - } - } - return false -} - -// DownloadFile downloads a file and verifies its checksum. -func (db *ChecksumDB) DownloadFile(url, dstPath string) error { - if err := db.Verify(dstPath); err == nil { - fmt.Printf("%s is up-to-date\n", dstPath) - return nil - } - fmt.Printf("%s is stale\n", dstPath) - fmt.Printf("downloading from %s\n", url) - - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("download error: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: status %d", resp.StatusCode) - } - if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { - return err - } - fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return err - } - dst := newDownloadWriter(fd, resp.ContentLength) - _, err = io.Copy(dst, resp.Body) - dst.Close() - if err != nil { - return err - } - return db.Verify(dstPath) -} - -type downloadWriter struct { - file *os.File - dstBuf *bufio.Writer - size int64 - written int64 - lastpct int64 -} - -func newDownloadWriter(dst *os.File, size int64) *downloadWriter { - return &downloadWriter{ - file: dst, - dstBuf: bufio.NewWriter(dst), - size: size, - } -} - -func (w *downloadWriter) Write(buf []byte) (int, error) { - n, err := w.dstBuf.Write(buf) - - // Report progress. - w.written += int64(n) - pct := w.written * 10 / w.size * 10 - if pct != w.lastpct { - if w.lastpct != 0 { - fmt.Print("...") - } - fmt.Print(pct, "%") - w.lastpct = pct - } - return n, err -} - -func (w *downloadWriter) Close() error { - if w.lastpct > 0 { - fmt.Println() // Finish the progress line. - } - flushErr := w.dstBuf.Flush() - closeErr := w.file.Close() - if flushErr != nil { - return flushErr - } - return closeErr -} diff --git a/internal/build/gotool.go b/internal/build/gotool.go index 2a47460418..172fa13464 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -24,6 +24,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/ethereum/go-ethereum/internal/download" ) type GoToolchain struct { @@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { // DownloadGo downloads the Go binary distribution and unpacks it into a temporary // directory. It returns the GOROOT of the unpacked toolchain. -func DownloadGo(csdb *ChecksumDB) string { - version, err := Version(csdb, "golang") +func DownloadGo(csdb *download.ChecksumDB) string { + version, err := csdb.FindVersion("golang") if err != nil { log.Fatal(err) } @@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string { } return goroot } - -// Version returns the versions defined in the checksumdb. -func Version(csdb *ChecksumDB, version string) (string, error) { - for _, l := range csdb.allChecksums { - if !strings.HasPrefix(l, "# version:") { - continue - } - v := strings.Split(l, ":")[1] - parts := strings.Split(v, " ") - if len(parts) != 2 { - log.Print("Erroneous version-string", "v", l) - continue - } - if parts[0] == version { - return parts[1], nil - } - } - return "", fmt.Errorf("no version found for '%v'", version) -} - -// DownloadAndVerifyChecksums downloads all files and checks that they match -// the checksum given in checksums.txt. -// This task can be used to sanity-check new checksums. -func DownloadAndVerifyChecksums(csdb *ChecksumDB) { - var ( - base = "" - ucache = os.TempDir() - ) - for _, l := range csdb.allChecksums { - if strings.HasPrefix(l, "# https://") { - base = l[2:] - continue - } - if strings.HasPrefix(l, "#") { - continue - } - hashFile := strings.Split(l, " ") - if len(hashFile) != 2 { - continue - } - file := hashFile[1] - url := base + file - dst := filepath.Join(ucache, file) - if err := csdb.DownloadFile(url, dst); err != nil { - log.Print(err) - } - } -} diff --git a/internal/download/download.go b/internal/download/download.go new file mode 100644 index 0000000000..f388350607 --- /dev/null +++ b/internal/download/download.go @@ -0,0 +1,261 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package download implements checksum-verified file downloads. +package download + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "slices" + "strings" +) + +// ChecksumDB keeps file checksums and tool versions. +type ChecksumDB struct { + hashes []hashEntry + versions []versionEntry +} + +type versionEntry struct { + name string + version string +} + +type urlEntry struct { + url string +} + +type hashEntry struct { + hash string + file string + u *urlEntry +} + +// MustLoadChecksums loads a file containing checksums. +func MustLoadChecksums(file string) *ChecksumDB { + content, err := os.ReadFile(file) + if err != nil { + log.Fatal("can't load checksum file: " + err.Error()) + } + db, err := ParseChecksums(content) + if err != nil { + log.Fatalf("invalid checksums in %s: %v", file, err) + } + return db +} + +// ParseChecksums parses a checksum database. +func ParseChecksums(input []byte) (*ChecksumDB, error) { + var ( + csdb = new(ChecksumDB) + rd = bytes.NewBuffer(input) + lastURL *urlEntry + ) + for lineNum := 1; ; lineNum++ { + line, err := rd.ReadString('\n') + if err == io.EOF { + break + } + line = strings.TrimSpace(line) + switch { + case line == "": + // Blank lines are allowed, and they reset the current urlEntry. + lastURL = nil + + case strings.HasPrefix(line, "#"): + // It's a comment. Some comments have special meaning. + content := strings.TrimLeft(line, "# ") + switch { + case strings.HasPrefix(content, "version:"): + // Version comments define the version of a tool. + v := strings.Split(content, ":")[1] + parts := strings.Split(v, " ") + if len(parts) != 2 { + return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v) + } + csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]}) + + case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"): + // URL comments define the URL where the following files are found. Here + // we keep track of the last found urlEntry and attach it to each file later. + lastURL = &urlEntry{content} + } + + default: + // It's a file hash entry. + fields := strings.Fields(line) + if len(fields) != 2 { + return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields)) + } + csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL}) + } + } + return csdb, nil +} + +// Verify checks whether the given file is valid according to the checksum database. +func (db *ChecksumDB) Verify(path string) error { + fd, err := os.Open(path) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { + return err + } + fileHash := hex.EncodeToString(h.Sum(nil)) + if !db.containsHash(filepath.Base(path), fileHash) { + return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path)) + } + return nil +} + +// DownloadFile downloads a file and verifies its checksum. +func (db *ChecksumDB) DownloadFile(url, dstPath string) error { + if err := db.Verify(dstPath); err == nil { + fmt.Printf("%s is up-to-date\n", dstPath) + return nil + } + fmt.Printf("%s is stale\n", dstPath) + fmt.Printf("downloading from %s\n", url) + + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("download error: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + dst := newDownloadWriter(fd, resp.ContentLength) + _, err = io.Copy(dst, resp.Body) + dst.Close() + if err != nil { + return err + } + return db.Verify(dstPath) +} + +// FindVersion returns the current known version of a tool, if it is defined in the file. +func (db *ChecksumDB) FindVersion(tool string) (string, error) { + for _, e := range db.versions { + if e.name == tool { + return e.version, nil + } + } + return "", fmt.Errorf("tool version %q not defined in checksum database", tool) +} + +// FindURL gets the URL for a file. +func (db *ChecksumDB) FindURL(basename string) (string, error) { + for _, e := range db.hashes { + if e.file == basename { + if e.u == nil { + return "", fmt.Errorf("file %q has no URL defined", e.file) + } + return e.u.url + e.file, nil + } + } + return "", fmt.Errorf("file %q does not exist in checksum database", basename) +} + +// VerifyAll downloads all files and checks that they match the checksum given in +// the database. This task can be used to sanity-check new checksums. +func (db *ChecksumDB) VerifyAll() { + var ( + tmp = os.TempDir() + ) + for _, e := range db.hashes { + if e.u == nil { + log.Printf("Skipping verification of %s: no URL defined in checksum database", e.file) + continue + } + url := e.u.url + e.file + dst := filepath.Join(tmp, e.file) + if err := db.DownloadFile(url, dst); err != nil { + log.Print(err) + } + } +} + +func (db *ChecksumDB) containsHash(basename, hash string) bool { + return slices.ContainsFunc(db.hashes, func(e hashEntry) bool { + return e.file == basename && e.hash == hash + }) +} + +type downloadWriter struct { + file *os.File + dstBuf *bufio.Writer + size int64 + written int64 + lastpct int64 +} + +func newDownloadWriter(dst *os.File, size int64) *downloadWriter { + return &downloadWriter{ + file: dst, + dstBuf: bufio.NewWriter(dst), + size: size, + } +} + +func (w *downloadWriter) Write(buf []byte) (int, error) { + n, err := w.dstBuf.Write(buf) + + // Report progress. + w.written += int64(n) + pct := w.written * 10 / w.size * 10 + if pct != w.lastpct { + if w.lastpct != 0 { + fmt.Print("...") + } + fmt.Print(pct, "%") + w.lastpct = pct + } + return n, err +} + +func (w *downloadWriter) Close() error { + if w.lastpct > 0 { + fmt.Println() // Finish the progress line. + } + flushErr := w.dstBuf.Flush() + closeErr := w.file.Close() + if flushErr != nil { + return flushErr + } + return closeErr +}