forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 06:36:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

internal/build: refactor file downloader into separate package

Browse source
This commit is contained in:
Felix Lange 2025-05-13 23:15:07 +02:00
parent c7bdf6b2fe
commit 13d72092b0
4 changed files with 282 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
build/ci.go
View file

@ -59,6 +59,7 @@ import (
"github.com/cespare/cp"
"github.com/ethereum/go-ethereum/crypto/signify"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/internal/download"
"github.com/ethereum/go-ethereum/internal/version"
)
@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
// Configure the toolchain.
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
if *dlgo {
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
tc.Root = build.DownloadGo(csdb)
}
// Disable CLI markdown doc generation in release builds.
@ -285,7 +286,7 @@ func doTest(cmdline []string) {
flag.CommandLine.Parse(cmdline)
// Get test fixtures.
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
downloadSpecTestFixtures(csdb, *cachedir)
// Configure the toolchain.
@ -329,8 +330,8 @@ func doTest(cmdline []string) {
}
// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
executionSpecTestsVersion, err := csdb.FindVersion("spec-tests")
if err != nil {
log.Fatal(err)
}
@ -444,8 +445,8 @@ func doLint(cmdline []string) {
// downloadLinter downloads and unpacks golangci-lint.
func downloadLinter(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "golangci")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("golangci")
if err != nil {
log.Fatal(err)
}
@ -497,8 +498,8 @@ func protocArchiveBaseName() (string, error) {
// in the generate command. It returns the full path of the directory
// containing the 'protoc-gen-go' executable.
func downloadProtocGenGo(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "protoc-gen-go")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("protoc-gen-go")
if err != nil {
log.Fatal(err)
}
@ -531,8 +532,8 @@ func downloadProtocGenGo(cachedir string) string {
// files as a CI step. It returns the full path to the directory containing
// the protoc executable.
func downloadProtoc(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "protoc")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("protoc")
if err != nil {
log.Fatal(err)
}
@ -826,11 +827,11 @@ func doDebianSource(cmdline []string) {
// downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
// to bootstrap the builder Go.
func downloadGoBootstrapSources(cachedir string) []string {
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
var bundles []string
for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
gobootVersion, err := build.Version(csdb, booter)
gobootVersion, err := csdb.FindVersion(booter)
if err != nil {
log.Fatal(err)
}
@ -847,8 +848,8 @@ func downloadGoBootstrapSources(cachedir string) []string {
// downloadGoSources downloads the Go source tarball.
func downloadGoSources(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
dlgoVersion, err := build.Version(csdb, "golang")
csdb := download.MustLoadChecksums("build/checksums.txt")
dlgoVersion, err := csdb.FindVersion("golang")
if err != nil {
log.Fatal(err)
}
@ -1181,5 +1182,6 @@ func doPurge(cmdline []string) {
}
func doSanityCheck() {
build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
csdb := download.MustLoadChecksums("build/checksums.txt")
csdb.VerifyAll()
}

151
internal/build/download.go
View file

@ -1,151 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package build
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
)
// ChecksumDB keeps file checksums.
type ChecksumDB struct {
allChecksums []string
}
// MustLoadChecksums loads a file containing checksums.
func MustLoadChecksums(file string) *ChecksumDB {
content, err := os.ReadFile(file)
if err != nil {
log.Fatal("can't load checksum file: " + err.Error())
}
return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
}
// Verify checks whether the given file is valid according to the checksum database.
func (db *ChecksumDB) Verify(path string) error {
fd, err := os.Open(path)
if err != nil {
return err
}
defer fd.Close()
h := sha256.New()
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
return err
}
fileHash := hex.EncodeToString(h.Sum(nil))
if !db.findHash(filepath.Base(path), fileHash) {
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
}
return nil
}
func (db *ChecksumDB) findHash(basename, hash string) bool {
want := hash + " " + basename
for _, line := range db.allChecksums {
if strings.TrimSpace(line) == want {
return true
}
}
return false
}
// DownloadFile downloads a file and verifies its checksum.
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
if err := db.Verify(dstPath); err == nil {
fmt.Printf("%s is up-to-date\n", dstPath)
return nil
}
fmt.Printf("%s is stale\n", dstPath)
fmt.Printf("downloading from %s\n", url)
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("download error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download error: status %d", resp.StatusCode)
}
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
return err
}
fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
dst := newDownloadWriter(fd, resp.ContentLength)
_, err = io.Copy(dst, resp.Body)
dst.Close()
if err != nil {
return err
}
return db.Verify(dstPath)
}
type downloadWriter struct {
file *os.File
dstBuf *bufio.Writer
size int64
written int64
lastpct int64
}
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
return &downloadWriter{
file: dst,
dstBuf: bufio.NewWriter(dst),
size: size,
}
}
func (w *downloadWriter) Write(buf []byte) (int, error) {
n, err := w.dstBuf.Write(buf)
// Report progress.
w.written += int64(n)
pct := w.written * 10 / w.size * 10
if pct != w.lastpct {
if w.lastpct != 0 {
fmt.Print("...")
}
fmt.Print(pct, "%")
w.lastpct = pct
}
return n, err
}
func (w *downloadWriter) Close() error {
if w.lastpct > 0 {
fmt.Println() // Finish the progress line.
}
flushErr := w.dstBuf.Flush()
closeErr := w.file.Close()
if flushErr != nil {
return flushErr
}
return closeErr
}

54
internal/build/gotool.go
View file

@ -24,6 +24,8 @@ import (
"path/filepath"
"runtime"
"strings"
"github.com/ethereum/go-ethereum/internal/download"
)
type GoToolchain struct {
@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
// DownloadGo downloads the Go binary distribution and unpacks it into a temporary
// directory. It returns the GOROOT of the unpacked toolchain.
func DownloadGo(csdb *ChecksumDB) string {
version, err := Version(csdb, "golang")
func DownloadGo(csdb *download.ChecksumDB) string {
version, err := csdb.FindVersion("golang")
if err != nil {
log.Fatal(err)
}
@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
}
return goroot
}
// Version returns the versions defined in the checksumdb.
func Version(csdb *ChecksumDB, version string) (string, error) {
for _, l := range csdb.allChecksums {
if !strings.HasPrefix(l, "# version:") {
continue
}
v := strings.Split(l, ":")[1]
parts := strings.Split(v, " ")
if len(parts) != 2 {
log.Print("Erroneous version-string", "v", l)
continue
}
if parts[0] == version {
return parts[1], nil
}
}
return "", fmt.Errorf("no version found for '%v'", version)
}
// DownloadAndVerifyChecksums downloads all files and checks that they match
// the checksum given in checksums.txt.
// This task can be used to sanity-check new checksums.
func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
var (
base = ""
ucache = os.TempDir()
)
for _, l := range csdb.allChecksums {
if strings.HasPrefix(l, "# https://") {
base = l[2:]
continue
}
if strings.HasPrefix(l, "#") {
continue
}
hashFile := strings.Split(l, " ")
if len(hashFile) != 2 {
continue
}
file := hashFile[1]
url := base + file
dst := filepath.Join(ucache, file)
if err := csdb.DownloadFile(url, dst); err != nil {
log.Print(err)
}
}
}

261
internal/download/download.go Normal file
View file

@ -0,0 +1,261 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package download implements checksum-verified file downloads.
package download
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"slices"
"strings"
)
// ChecksumDB keeps file checksums and tool versions.
type ChecksumDB struct {
hashes []hashEntry
versions []versionEntry
}
type versionEntry struct {
name string
version string
}
type urlEntry struct {
url string
}
type hashEntry struct {
hash string
file string
u *urlEntry
}
// MustLoadChecksums loads a file containing checksums.
func MustLoadChecksums(file string) *ChecksumDB {
content, err := os.ReadFile(file)
if err != nil {
log.Fatal("can't load checksum file: " + err.Error())
}
db, err := ParseChecksums(content)
if err != nil {
log.Fatalf("invalid checksums in %s: %v", file, err)
}
return db
}
// ParseChecksums parses a checksum database.
func ParseChecksums(input []byte) (*ChecksumDB, error) {
var (
csdb = new(ChecksumDB)
rd = bytes.NewBuffer(input)
lastURL *urlEntry
)
for lineNum := 1; ; lineNum++ {
line, err := rd.ReadString('\n')
if err == io.EOF {
break
}
line = strings.TrimSpace(line)
switch {
case line == "":
// Blank lines are allowed, and they reset the current urlEntry.
lastURL = nil
case strings.HasPrefix(line, "#"):
// It's a comment. Some comments have special meaning.
content := strings.TrimLeft(line, "# ")
switch {
case strings.HasPrefix(content, "version:"):
// Version comments define the version of a tool.
v := strings.Split(content, ":")[1]
parts := strings.Split(v, " ")
if len(parts) != 2 {
return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
}
csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
// URL comments define the URL where the following files are found. Here
// we keep track of the last found urlEntry and attach it to each file later.
lastURL = &urlEntry{content}
}
default:
// It's a file hash entry.
fields := strings.Fields(line)
if len(fields) != 2 {
return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
}
csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
}
}
return csdb, nil
}
// Verify checks whether the given file is valid according to the checksum database.
func (db *ChecksumDB) Verify(path string) error {
fd, err := os.Open(path)
if err != nil {
return err
}
defer fd.Close()
h := sha256.New()
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
return err
}
fileHash := hex.EncodeToString(h.Sum(nil))
if !db.containsHash(filepath.Base(path), fileHash) {
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
}
return nil
}
// DownloadFile downloads a file and verifies its checksum.
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
if err := db.Verify(dstPath); err == nil {
fmt.Printf("%s is up-to-date\n", dstPath)
return nil
}
fmt.Printf("%s is stale\n", dstPath)
fmt.Printf("downloading from %s\n", url)
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("download error: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download error: status %d", resp.StatusCode)
}
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
return err
}
fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
dst := newDownloadWriter(fd, resp.ContentLength)
_, err = io.Copy(dst, resp.Body)
dst.Close()
if err != nil {
return err
}
return db.Verify(dstPath)
}
// FindVersion returns the current known version of a tool, if it is defined in the file.
func (db *ChecksumDB) FindVersion(tool string) (string, error) {
for _, e := range db.versions {
if e.name == tool {
return e.version, nil
}
}
return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
}
// FindURL gets the URL for a file.
func (db *ChecksumDB) FindURL(basename string) (string, error) {
for _, e := range db.hashes {
if e.file == basename {
if e.u == nil {
return "", fmt.Errorf("file %q has no URL defined", e.file)
}
return e.u.url + e.file, nil
}
}
return "", fmt.Errorf("file %q does not exist in checksum database", basename)
}
// VerifyAll downloads all files and checks that they match the checksum given in
// the database. This task can be used to sanity-check new checksums.
func (db *ChecksumDB) VerifyAll() {
var (
tmp = os.TempDir()
)
for _, e := range db.hashes {
if e.u == nil {
log.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
continue
}
url := e.u.url + e.file
dst := filepath.Join(tmp, e.file)
if err := db.DownloadFile(url, dst); err != nil {
log.Print(err)
}
}
}
func (db *ChecksumDB) containsHash(basename, hash string) bool {
return slices.ContainsFunc(db.hashes, func(e hashEntry) bool {
return e.file == basename && e.hash == hash
})
}
type downloadWriter struct {
file *os.File
dstBuf *bufio.Writer
size int64
written int64
lastpct int64
}
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
return &downloadWriter{
file: dst,
dstBuf: bufio.NewWriter(dst),
size: size,
}
}
func (w *downloadWriter) Write(buf []byte) (int, error) {
n, err := w.dstBuf.Write(buf)
// Report progress.
w.written += int64(n)
pct := w.written * 10 / w.size * 10
if pct != w.lastpct {
if w.lastpct != 0 {
fmt.Print("...")
}
fmt.Print(pct, "%")
w.lastpct = pct
}
return n, err
}
func (w *downloadWriter) Close() error {
if w.lastpct > 0 {
fmt.Println() // Finish the progress line.
}
flushErr := w.dstBuf.Flush()
closeErr := w.file.Close()
if flushErr != nil {
return flushErr
}
return closeErr
}