internal/build: refactor file downloader into separate package

build/ci.go

@@ -59,6 +59,7 @@ import (
 	"github.com/cespare/cp"
 	"github.com/ethereum/go-ethereum/crypto/signify"
 	"github.com/ethereum/go-ethereum/internal/build"
+	"github.com/ethereum/go-ethereum/internal/download"
 	"github.com/ethereum/go-ethereum/internal/version"
 )
 
@@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		csdb := build.MustLoadChecksums("build/checksums.txt")
+		csdb := download.MustLoadChecksums("build/checksums.txt")
 		tc.Root = build.DownloadGo(csdb)
 	}
 	// Disable CLI markdown doc generation in release builds.
@@ -285,7 +286,7 @@ func doTest(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 
 	// Get test fixtures.
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
 	downloadSpecTestFixtures(csdb, *cachedir)
 
 	// Configure the toolchain.
@@ -329,8 +330,8 @@ func doTest(cmdline []string) {
 }
 
 // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
-func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
-	executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
+	executionSpecTestsVersion, err := csdb.FindVersion("spec-tests")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -444,8 +445,8 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "golangci")
+	version, err := csdb.FindVersion("golangci")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -497,8 +498,8 @@ func protocArchiveBaseName() (string, error) {
 // in the generate command.  It returns the full path of the directory
 // containing the 'protoc-gen-go' executable.
 func downloadProtocGenGo(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc-gen-go")
+	version, err := csdb.FindVersion("protoc-gen-go")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -531,8 +532,8 @@ func downloadProtocGenGo(cachedir string) string {
 // files as a CI step.  It returns the full path to the directory containing
 // the protoc executable.
 func downloadProtoc(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc")
+	version, err := csdb.FindVersion("protoc")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -826,11 +827,11 @@ func doDebianSource(cmdline []string) {
 // downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
 // to bootstrap the builder Go.
 func downloadGoBootstrapSources(cachedir string) []string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
 
 	var bundles []string
 	for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
-		gobootVersion, err := build.Version(csdb, booter)
+		gobootVersion, err := csdb.FindVersion(booter)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -847,8 +848,8 @@ func downloadGoBootstrapSources(cachedir string) []string {
 
 // downloadGoSources downloads the Go source tarball.
 func downloadGoSources(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	dlgoVersion, err := build.Version(csdb, "golang")
+	dlgoVersion, err := csdb.FindVersion("golang")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -1181,5 +1182,6 @@ func doPurge(cmdline []string) {
 }
 
 func doSanityCheck() {
-	build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
+	csdb := download.MustLoadChecksums("build/checksums.txt")
+	csdb.VerifyAll()
 }

internal/build/download.go

@@ -1,151 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package build
-
-import (
-	"bufio"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// ChecksumDB keeps file checksums.
-type ChecksumDB struct {
-	allChecksums []string
-}
-
-// MustLoadChecksums loads a file containing checksums.
-func MustLoadChecksums(file string) *ChecksumDB {
-	content, err := os.ReadFile(file)
-	if err != nil {
-		log.Fatal("can't load checksum file: " + err.Error())
-	}
-	return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
-}
-
-// Verify checks whether the given file is valid according to the checksum database.
-func (db *ChecksumDB) Verify(path string) error {
-	fd, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
-		return err
-	}
-	fileHash := hex.EncodeToString(h.Sum(nil))
-	if !db.findHash(filepath.Base(path), fileHash) {
-		return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
-	}
-	return nil
-}
-
-func (db *ChecksumDB) findHash(basename, hash string) bool {
-	want := hash + "  " + basename
-	for _, line := range db.allChecksums {
-		if strings.TrimSpace(line) == want {
-			return true
-		}
-	}
-	return false
-}
-
-// DownloadFile downloads a file and verifies its checksum.
-func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
-	if err := db.Verify(dstPath); err == nil {
-		fmt.Printf("%s is up-to-date\n", dstPath)
-		return nil
-	}
-	fmt.Printf("%s is stale\n", dstPath)
-	fmt.Printf("downloading from %s\n", url)
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return fmt.Errorf("download error: %v", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("download error: status %d", resp.StatusCode)
-	}
-	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
-		return err
-	}
-	fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	dst := newDownloadWriter(fd, resp.ContentLength)
-	_, err = io.Copy(dst, resp.Body)
-	dst.Close()
-	if err != nil {
-		return err
-	}
-	return db.Verify(dstPath)
-}
-
-type downloadWriter struct {
-	file    *os.File
-	dstBuf  *bufio.Writer
-	size    int64
-	written int64
-	lastpct int64
-}
-
-func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
-	return &downloadWriter{
-		file:   dst,
-		dstBuf: bufio.NewWriter(dst),
-		size:   size,
-	}
-}
-
-func (w *downloadWriter) Write(buf []byte) (int, error) {
-	n, err := w.dstBuf.Write(buf)
-
-	// Report progress.
-	w.written += int64(n)
-	pct := w.written * 10 / w.size * 10
-	if pct != w.lastpct {
-		if w.lastpct != 0 {
-			fmt.Print("...")
-		}
-		fmt.Print(pct, "%")
-		w.lastpct = pct
-	}
-	return n, err
-}
-
-func (w *downloadWriter) Close() error {
-	if w.lastpct > 0 {
-		fmt.Println() // Finish the progress line.
-	}
-	flushErr := w.dstBuf.Flush()
-	closeErr := w.file.Close()
-	if flushErr != nil {
-		return flushErr
-	}
-	return closeErr
-}

internal/build/gotool.go

@@ -24,6 +24,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"github.com/ethereum/go-ethereum/internal/download"
 )
 
 type GoToolchain struct {
@@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
 
 // DownloadGo downloads the Go binary distribution and unpacks it into a temporary
 // directory. It returns the GOROOT of the unpacked toolchain.
-func DownloadGo(csdb *ChecksumDB) string {
+func DownloadGo(csdb *download.ChecksumDB) string {
-	version, err := Version(csdb, "golang")
+	version, err := csdb.FindVersion("golang")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
 	}
 	return goroot
 }
-
-// Version returns the versions defined in the checksumdb.
-func Version(csdb *ChecksumDB, version string) (string, error) {
-	for _, l := range csdb.allChecksums {
-		if !strings.HasPrefix(l, "# version:") {
-			continue
-		}
-		v := strings.Split(l, ":")[1]
-		parts := strings.Split(v, " ")
-		if len(parts) != 2 {
-			log.Print("Erroneous version-string", "v", l)
-			continue
-		}
-		if parts[0] == version {
-			return parts[1], nil
-		}
-	}
-	return "", fmt.Errorf("no version found for '%v'", version)
-}
-
-// DownloadAndVerifyChecksums downloads all files and checks that they match
-// the checksum given in checksums.txt.
-// This task can be used to sanity-check new checksums.
-func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
-	var (
-		base   = ""
-		ucache = os.TempDir()
-	)
-	for _, l := range csdb.allChecksums {
-		if strings.HasPrefix(l, "# https://") {
-			base = l[2:]
-			continue
-		}
-		if strings.HasPrefix(l, "#") {
-			continue
-		}
-		hashFile := strings.Split(l, "  ")
-		if len(hashFile) != 2 {
-			continue
-		}
-		file := hashFile[1]
-		url := base + file
-		dst := filepath.Join(ucache, file)
-		if err := csdb.DownloadFile(url, dst); err != nil {
-			log.Print(err)
-		}
-	}
-}

internal/download/download.go

@@ -0,0 +1,261 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package download implements checksum-verified file downloads.
+package download
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+)
+
+// ChecksumDB keeps file checksums and tool versions.
+type ChecksumDB struct {
+	hashes   []hashEntry
+	versions []versionEntry
+}
+
+type versionEntry struct {
+	name    string
+	version string
+}
+
+type urlEntry struct {
+	url string
+}
+
+type hashEntry struct {
+	hash string
+	file string
+	u    *urlEntry
+}
+
+// MustLoadChecksums loads a file containing checksums.
+func MustLoadChecksums(file string) *ChecksumDB {
+	content, err := os.ReadFile(file)
+	if err != nil {
+		log.Fatal("can't load checksum file: " + err.Error())
+	}
+	db, err := ParseChecksums(content)
+	if err != nil {
+		log.Fatalf("invalid checksums in %s: %v", file, err)
+	}
+	return db
+}
+
+// ParseChecksums parses a checksum database.
+func ParseChecksums(input []byte) (*ChecksumDB, error) {
+	var (
+		csdb    = new(ChecksumDB)
+		rd      = bytes.NewBuffer(input)
+		lastURL *urlEntry
+	)
+	for lineNum := 1; ; lineNum++ {
+		line, err := rd.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		line = strings.TrimSpace(line)
+		switch {
+		case line == "":
+			// Blank lines are allowed, and they reset the current urlEntry.
+			lastURL = nil
+
+		case strings.HasPrefix(line, "#"):
+			// It's a comment. Some comments have special meaning.
+			content := strings.TrimLeft(line, "# ")
+			switch {
+			case strings.HasPrefix(content, "version:"):
+				// Version comments define the version of a tool.
+				v := strings.Split(content, ":")[1]
+				parts := strings.Split(v, " ")
+				if len(parts) != 2 {
+					return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
+				}
+				csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
+
+			case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
+				// URL comments define the URL where the following files are found. Here
+				// we keep track of the last found urlEntry and attach it to each file later.
+				lastURL = &urlEntry{content}
+			}
+
+		default:
+			// It's a file hash entry.
+			fields := strings.Fields(line)
+			if len(fields) != 2 {
+				return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
+			}
+			csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
+		}
+	}
+	return csdb, nil
+}
+
+// Verify checks whether the given file is valid according to the checksum database.
+func (db *ChecksumDB) Verify(path string) error {
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
+		return err
+	}
+	fileHash := hex.EncodeToString(h.Sum(nil))
+	if !db.containsHash(filepath.Base(path), fileHash) {
+		return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
+	}
+	return nil
+}
+
+// DownloadFile downloads a file and verifies its checksum.
+func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
+	if err := db.Verify(dstPath); err == nil {
+		fmt.Printf("%s is up-to-date\n", dstPath)
+		return nil
+	}
+	fmt.Printf("%s is stale\n", dstPath)
+	fmt.Printf("downloading from %s\n", url)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("download error: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("download error: status %d", resp.StatusCode)
+	}
+	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+		return err
+	}
+	fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	dst := newDownloadWriter(fd, resp.ContentLength)
+	_, err = io.Copy(dst, resp.Body)
+	dst.Close()
+	if err != nil {
+		return err
+	}
+	return db.Verify(dstPath)
+}
+
+// FindVersion returns the current known version of a tool, if it is defined in the file.
+func (db *ChecksumDB) FindVersion(tool string) (string, error) {
+	for _, e := range db.versions {
+		if e.name == tool {
+			return e.version, nil
+		}
+	}
+	return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
+}
+
+// FindURL gets the URL for a file.
+func (db *ChecksumDB) FindURL(basename string) (string, error) {
+	for _, e := range db.hashes {
+		if e.file == basename {
+			if e.u == nil {
+				return "", fmt.Errorf("file %q has no URL defined", e.file)
+			}
+			return e.u.url + e.file, nil
+		}
+	}
+	return "", fmt.Errorf("file %q does not exist in checksum database", basename)
+}
+
+// VerifyAll downloads all files and checks that they match the checksum given in
+// the database. This task can be used to sanity-check new checksums.
+func (db *ChecksumDB) VerifyAll() {
+	var (
+		tmp = os.TempDir()
+	)
+	for _, e := range db.hashes {
+		if e.u == nil {
+			log.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
+			continue
+		}
+		url := e.u.url + e.file
+		dst := filepath.Join(tmp, e.file)
+		if err := db.DownloadFile(url, dst); err != nil {
+			log.Print(err)
+		}
+	}
+}
+
+func (db *ChecksumDB) containsHash(basename, hash string) bool {
+	return slices.ContainsFunc(db.hashes, func(e hashEntry) bool {
+		return e.file == basename && e.hash == hash
+	})
+}
+
+type downloadWriter struct {
+	file    *os.File
+	dstBuf  *bufio.Writer
+	size    int64
+	written int64
+	lastpct int64
+}
+
+func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
+	return &downloadWriter{
+		file:   dst,
+		dstBuf: bufio.NewWriter(dst),
+		size:   size,
+	}
+}
+
+func (w *downloadWriter) Write(buf []byte) (int, error) {
+	n, err := w.dstBuf.Write(buf)
+
+	// Report progress.
+	w.written += int64(n)
+	pct := w.written * 10 / w.size * 10
+	if pct != w.lastpct {
+		if w.lastpct != 0 {
+			fmt.Print("...")
+		}
+		fmt.Print(pct, "%")
+		w.lastpct = pct
+	}
+	return n, err
+}
+
+func (w *downloadWriter) Close() error {
+	if w.lastpct > 0 {
+		fmt.Println() // Finish the progress line.
+	}
+	flushErr := w.dstBuf.Flush()
+	closeErr := w.file.Close()
+	if flushErr != nil {
+		return flushErr
+	}
+	return closeErr
+}