// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . // Package download implements checksum-verified file downloads. package download import ( "bufio" "bytes" "crypto/sha256" "encoding/hex" "fmt" "io" "iter" "log" "net/http" "net/url" "os" "path/filepath" "slices" "strings" ) // ChecksumDB keeps file checksums and tool versions. type ChecksumDB struct { hashes []hashEntry versions []versionEntry } type versionEntry struct { name string version string } type hashEntry struct { hash string file string url *url.URL } // MustLoadChecksums loads a file containing checksums. func MustLoadChecksums(file string) *ChecksumDB { content, err := os.ReadFile(file) if err != nil { log.Fatal("can't load checksum file: " + err.Error()) } db, err := ParseChecksums(content) if err != nil { log.Fatalf("invalid checksums in %s: %v", file, err) } return db } // ParseChecksums parses a checksum database. func ParseChecksums(input []byte) (*ChecksumDB, error) { var ( csdb = new(ChecksumDB) rd = bytes.NewBuffer(input) lastURL *url.URL ) for lineNum := 1; ; lineNum++ { line, err := rd.ReadString('\n') if err == io.EOF { break } line = strings.TrimSpace(line) switch { case line == "": // Blank lines are allowed, and they reset the current urlEntry. lastURL = nil case strings.HasPrefix(line, "#"): // It's a comment. Some comments have special meaning. content := strings.TrimLeft(line, "# ") switch { case strings.HasPrefix(content, "version:"): // Version comments define the version of a tool. v := strings.Split(content, ":")[1] parts := strings.Split(v, " ") if len(parts) != 2 { return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v) } csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]}) case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"): // URL comments define the URL where the following files are found. Here // we keep track of the last found urlEntry and attach it to each file later. u, err := url.Parse(content) if err != nil { return nil, fmt.Errorf("line %d: invalid URL: %v", lineNum, err) } lastURL = u } default: // It's a file hash entry. fields := strings.Fields(line) if len(fields) != 2 { return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields)) } csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL}) } } return csdb, nil } // Files returns an iterator over all file names. func (db *ChecksumDB) Files() iter.Seq[string] { return func(yield func(string) bool) { for _, e := range db.hashes { if !yield(e.file) { return } } } } // Verify checks whether the given file is valid according to the checksum database. func (db *ChecksumDB) Verify(path string) error { fd, err := os.Open(path) if err != nil { return err } defer fd.Close() h := sha256.New() if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { return err } fileHash := hex.EncodeToString(h.Sum(nil)) if !db.containsHash(filepath.Base(path), fileHash) { return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path)) } return nil } // DownloadFile downloads a file and verifies its checksum. func (db *ChecksumDB) DownloadFile(url, dstPath string) error { if err := db.Verify(dstPath); err == nil { fmt.Printf("%s is up-to-date\n", dstPath) return nil } fmt.Printf("%s is stale\n", dstPath) fmt.Printf("downloading from %s\n", url) resp, err := http.Get(url) if err != nil { return fmt.Errorf("download error: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("download error: status %d", resp.StatusCode) } if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return err } fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) if err != nil { return err } dst := newDownloadWriter(fd, resp.ContentLength) _, err = io.Copy(dst, resp.Body) dst.Close() if err != nil { return err } return db.Verify(dstPath) } // FindVersion returns the current known version of a tool, if it is defined in the file. func (db *ChecksumDB) FindVersion(tool string) (string, error) { for _, e := range db.versions { if e.name == tool { return e.version, nil } } return "", fmt.Errorf("tool version %q not defined in checksum database", tool) } // FindURL gets the URL for a file. func (db *ChecksumDB) FindURL(basename string) (string, error) { for _, e := range db.hashes { if e.file == basename { if e.url == nil { return "", fmt.Errorf("file %q has no URL defined", e.file) } return e.url.JoinPath(e.file).String(), nil } } return "", fmt.Errorf("file %q does not exist in checksum database", basename) } // VerifyAll downloads all files and checks that they match the checksum given in // the database. This task can be used to sanity-check new checksums. func (db *ChecksumDB) VerifyAll() { var ( tmp = os.TempDir() ) for _, e := range db.hashes { if e.url == nil { log.Printf("Skipping verification of %s: no URL defined in checksum database", e.file) continue } url := e.url.JoinPath(e.file).String() dst := filepath.Join(tmp, e.file) if err := db.DownloadFile(url, dst); err != nil { log.Print(err) } } } func (db *ChecksumDB) containsHash(basename, hash string) bool { return slices.ContainsFunc(db.hashes, func(e hashEntry) bool { return e.file == basename && e.hash == hash }) } type downloadWriter struct { file *os.File dstBuf *bufio.Writer size int64 written int64 lastpct int64 } func newDownloadWriter(dst *os.File, size int64) *downloadWriter { return &downloadWriter{ file: dst, dstBuf: bufio.NewWriter(dst), size: size, } } func (w *downloadWriter) Write(buf []byte) (int, error) { n, err := w.dstBuf.Write(buf) // Report progress. w.written += int64(n) pct := w.written * 10 / w.size * 10 if pct != w.lastpct { if w.lastpct != 0 { fmt.Print("...") } fmt.Print(pct, "%") w.lastpct = pct } return n, err } func (w *downloadWriter) Close() error { if w.lastpct > 0 { fmt.Println() // Finish the progress line. } flushErr := w.dstBuf.Flush() closeErr := w.file.Close() if flushErr != nil { return flushErr } return closeErr }