build, internal, Makefile: upgrade package build (#1746)

Makefile

@@ -45,7 +45,7 @@ all:
 test: all
 	go run build/ci.go test -failfast
 
-#? quick-test: Run the tests except time-consuming tests.
+#? quick-test: Run the tests except time-consuming packages.
 quick-test: all
 	go run build/ci.go test --quick -failfast
 
@@ -53,14 +53,18 @@ quick-test: all
 lint: ## Run linters.
 	$(GORUN) build/ci.go lint
 
-#? tidy: Verify go.mod and go.sum by 'go mod tidy'
+#? tidy: Verify go.mod and go.sum are updated.
 tidy: ## Run 'go mod tidy'.
 	$(GORUN) build/ci.go tidy
 
-#? generate: Verify everything is 'go generate'-ed
+#? generate: Verify everything is 'go generate'.
 generate: ## Run 'go generate ./...'.
 	$(GORUN) build/ci.go generate
 
+#? baddeps: Verify certain dependencies are avoided.
+baddeps:
+	$(GORUN) build/ci.go baddeps
+
 #? fmt: Ensure consistent code formatting.
 fmt:
 	gofmt -s -w $(shell find . -name "*.go")

build/ci.go

@@ -24,14 +24,15 @@ Usage: go run build/ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-	lint        -- runs certain pre-selected linters
+	lint      -- runs certain pre-selected linters
-	tidy        -- verifies that everything is 'go mod tidy'-ed
+	tidy      -- verifies that everything is 'go mod tidy'-ed
-	generate    -- verifies that everything is 'go generate'-ed
+	generate  -- verifies that everything is 'go generate'-ed
+	baddeps   -- verifies that certain dependencies are avoided
 
-	install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
+	install    [ -arch architecture ] [ -cc compiler ] [ packages... ]  -- builds packages and executables
-	test       [ -coverage ] [ packages... ]                                                    -- runs the tests
+	test       [ -coverage ] [ packages... ]                            -- runs the tests
-	importkeys                                                                                  -- imports signing keys from env
+	importkeys                                                          -- imports signing keys from env
-	xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+	xgo        [ -alltools ] [ options ]                                -- cross builds according to options
 
 For all commands, -n prevents execution of external programs (dry run mode).
 */
@@ -40,31 +41,36 @@ package main
 import (
 	"flag"
 	"fmt"
-	"go/parser"
-	"go/token"
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strings"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/internal/build"
+	"github.com/XinFinOrg/XDPoSChain/internal/download"
 )
 
 var (
+	goModules = []string{
+		".",
+	}
+
 	// Files that end up in the geth-alltools*.zip archive.
 	allToolsArchiveFiles = []string{
 		"COPYING",
 		executablePath("abigen"),
 		executablePath("bootnode"),
+		executablePath("ethkey"),
 		executablePath("evm"),
-		executablePath("geth"),
+		executablePath("p2psim"),
 		executablePath("puppeth"),
 		executablePath("rlpdump"),
-		executablePath("swarm"),
+		executablePath("XDC"),
-		executablePath("wnode"),
 	}
 )
 
@@ -97,6 +103,8 @@ func main() {
 		doTidy()
 	case "generate":
 		doGenerate()
+	case "baddeps":
+		doBadDeps()
 	case "xgo":
 		doXgo(os.Args[2:])
 	default:
@@ -108,145 +116,131 @@ func main() {
 
 func doInstall(cmdline []string) {
 	var (
-		arch = flag.String("arch", "", "Architecture to cross build for")
+		dlgo       = flag.Bool("dlgo", false, "Download Go and build with it")
-		cc   = flag.String("cc", "", "C compiler to cross build with")
+		arch       = flag.String("arch", "", "Architecture to cross build for")
+		cc         = flag.String("cc", "", "C compiler to cross build with")
+		staticlink = flag.Bool("static", false, "Create statically-linked executable")
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
-	// Check Go version. People regularly open issues about compilation
+	// Configure the toolchain.
-	// failure with outdated Go. This should save them the trouble.
+	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
-	if !strings.Contains(runtime.Version(), "devel") {
+	if *dlgo {
-		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
+		csdb := download.MustLoadChecksums("build/checksums.txt")
-		var minor int
+		tc.Root = build.DownloadGo(csdb)
-		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
+	}
+	// Disable CLI markdown doc generation in release builds.
+	buildTags := []string{"urfave_cli_no_docs"}
 
-		if minor < 25 {
+	// Configure the build.
-			log.Println("You have Go version", runtime.Version())
+	gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...)
-			log.Println("XDC requires at least Go version 1.25 and cannot")
-			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
-			os.Exit(1)
-		}
-	}
-	// Compile packages given as arguments, or everything if there are no arguments.
-	packages := []string{"./..."}
-	if flag.NArg() > 0 {
-		packages = flag.Args()
-	}
-	// packages = build.ExpandPackagesNoVendor(packages)
 
-	if *arch == "" || *arch == runtime.GOARCH {
+	// Show packages during build.
-		goinstall := goTool("install", buildFlags(env)...)
+	gobuild.Args = append(gobuild.Args, "-v")
-		goinstall.Args = append(goinstall.Args, "-v")
-		goinstall.Args = append(goinstall.Args, packages...)
-		build.MustRun(goinstall)
-		return
-	}
-	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
-	if *arch == "arm" {
-		os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
-		for _, path := range filepath.SplitList(build.GOPATH()) {
-			os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
-		}
-	}
-	// Seems we are cross compiling, work around forbidden GOBIN
-	goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
-	goinstall.Args = append(goinstall.Args, "-v")
-	goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
-	goinstall.Args = append(goinstall.Args, packages...)
-	build.MustRun(goinstall)
 
-	if cmds, err := os.ReadDir("cmd"); err == nil {
+	// Now we choose what we're even building.
-		for _, cmd := range cmds {
+	// Default: collect all 'main' packages in cmd/ and build those.
-			pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
+	packages := flag.Args()
-			if err != nil {
+	if len(packages) == 0 {
-				log.Fatal(err)
+		// NOTE: to collect all main packages, use:
-			}
+		// packages = build.FindMainPackages(&tc, "./...")
-			for name := range pkgs {
+		packages = build.FindMainPackages(&tc, "./cmd/...")
-				if name == "main" {
+	}
-					gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
+
-					gobuild.Args = append(gobuild.Args, "-v")
+	// Do the build!
-					gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
+	for _, pkg := range packages {
-					gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
+		args := slices.Clone(gobuild.Args)
-					build.MustRun(gobuild)
+		args = append(args, "-o", executablePath(path.Base(pkg)))
-					break
+		args = append(args, pkg)
-				}
+		build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env})
-			}
-		}
 	}
 }
 
-func buildFlags(env build.Environment) (flags []string) {
+// buildFlags returns the go tool flags for building.
+func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) {
 	var ld []string
+	// See https://github.com/golang/go/issues/33772#issuecomment-528176001
+	// We need to set --buildid to the linker here, and also pass --build-id to the
+	// cgo-linker further down.
+	ld = append(ld, "--buildid=none")
 	if env.Commit != "" {
 		ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitCommit="+env.Commit)
 		ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitDate="+env.Date)
 	}
+	// Strip DWARF on darwin. This used to be required for certain things,
+	// and there is no downside to this, so we just keep doing it.
 	if runtime.GOOS == "darwin" {
 		ld = append(ld, "-s")
 	}
-
+	if runtime.GOOS == "linux" {
+		// Enforce the stacksize to 8M, which is the case on most platforms apart from
+		// alpine Linux.
+		// See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options
+		// regarding the options --build-id=none and --strip-all. It is needed for
+		// reproducible builds; removing references to temporary files in C-land, and
+		// making build-id reproducibly absent.
+		extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"}
+		if staticLinking {
+			extld = append(extld, "-static")
+			// Under static linking, use of certain glibc features must be
+			// disabled to avoid shared library dependencies.
+			buildTags = append(buildTags, "osusergo", "netgo")
+		}
+		ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'")
+	}
 	if len(ld) > 0 {
 		flags = append(flags, "-ldflags", strings.Join(ld, " "))
 	}
+	if len(buildTags) > 0 {
+		flags = append(flags, "-tags", strings.Join(buildTags, ","))
+	}
+	// We use -trimpath to avoid leaking local paths into the built executables.
+	flags = append(flags, "-trimpath")
 	return flags
 }
 
-func goTool(subcmd string, args ...string) *exec.Cmd {
-	return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
-}
-
-func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
-	cmd := build.GoTool(subcmd, args...)
-	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
-	if arch == "" || arch == runtime.GOARCH {
-		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
-	} else {
-		cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
-		cmd.Env = append(cmd.Env, "GOARCH="+arch)
-	}
-	if cc != "" {
-		cmd.Env = append(cmd.Env, "CC="+cc)
-	}
-	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
-			continue
-		}
-		cmd.Env = append(cmd.Env, e)
-	}
-	return cmd
-}
-
 // Running The Tests
 //
 // "tests" also includes static analysis tools such as vet.
-func doTest(cmdline []string) {
-	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
-	verbose := flag.Bool("v", false, "Whether to log verbosely")
-	quick := flag.Bool("quick", false, "Whether to skip long time test")
-	failfast := flag.Bool("failfast", false, "Do not start new tests after the first test failure")
-	flag.CommandLine.Parse(cmdline)
-	env := build.Env()
 
-	packages := []string{"./..."} // if a package has no test files, the lines in that package are not added to the total line count
+func doTest(cmdline []string) {
-	if len(flag.CommandLine.Args()) > 0 {
+	var (
-		packages = flag.CommandLine.Args()
+		dlgo     = flag.Bool("dlgo", false, "Download Go and build with it")
-	} else {
+		arch     = flag.String("arch", "", "Run tests for given architecture")
-		// added all files in all packages (except vendor) to coverage report files count, even there is no test file in the package
+		cc       = flag.String("cc", "", "Sets C compiler binary")
-		packages = build.ExpandPackages(packages, *quick)
+		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
+		verbose  = flag.Bool("v", false, "Whether to log verbosely")
+		race     = flag.Bool("race", false, "Execute the race detector")
+		short    = flag.Bool("short", false, "Pass the 'short'-flag to go test")
+		threads  = flag.Int("p", 1, "Number of CPU threads to use for testing")
+		quick    = flag.Bool("quick", false, "Whether to skip long time test")
+		failfast = flag.Bool("failfast", false, "Do not start new tests after the first test failure")
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	// Load checksums file (needed for both spec tests and dlgo)
+	csdb := download.MustLoadChecksums("build/checksums.txt")
+
+	// Configure the toolchain.
+	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
+	if *dlgo {
+		tc.Root = build.DownloadGo(csdb)
 	}
 
-	// Run analysis tools before the tests.
+	gotest := tc.Go("test")
-	// build.MustRun(goTool("vet", packages...))
+
+	// CI needs a bit more time for the statetests (default 45m).
+	gotest.Args = append(gotest.Args, "-timeout=45m")
+
+	// Enable integration-tests
+	gotest.Args = append(gotest.Args, "-tags=integrationtests")
 
-	// Run the actual tests.
-	// gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", fmt.Sprintf("%d", *threads))
-	gotest.Args = append(gotest.Args, "-p", "1")
 	if *coverage {
-		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover", "-coverprofile=coverage.txt")
+		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
 	if *verbose {
 		gotest.Args = append(gotest.Args, "-v")
@@ -254,27 +248,59 @@ func doTest(cmdline []string) {
 	if *failfast {
 		gotest.Args = append(gotest.Args, "-failfast")
 	}
+	if *race {
+		gotest.Args = append(gotest.Args, "-race")
+	}
+	if *short {
+		gotest.Args = append(gotest.Args, "-short")
+	}
 
+	packages := flag.CommandLine.Args()
+	if len(packages) > 0 {
+		if *quick {
+			packages = filterPackages(packages)
+		}
+		gotest.Args = append(gotest.Args, packages...)
+		build.MustRun(gotest)
+		return
+	}
+
+	// No packages specified, run all tests for all modules.
+	if *quick {
+		packages = filterPackages(build.FindAllPackages(&tc))
+	} else {
+		packages = []string{"./..."}
+	}
 	gotest.Args = append(gotest.Args, packages...)
-	build.MustRun(gotest)
+	for _, mod := range goModules {
+		test := *gotest
+		test.Dir = mod
+		build.MustRun(&test)
+	}
 }
 
-// doTidy assets that the Go modules files are tidied already.
+// filterPackages removes time-consuming packages.
+func filterPackages(packages []string) []string {
+	var filtered []string
+
+	for _, pkg := range packages {
+		if strings.Contains(pkg, "/consensus/tests/engine_v2_tests") {
+			continue
+		}
+		filtered = append(filtered, pkg)
+	}
+
+	return filtered
+}
+
+// doTidy runs go mod tidy check.
 func doTidy() {
-	targets := []string{"go.mod", "go.sum"}
+	var tc = new(build.GoToolchain)
 
-	hashes, err := build.HashFiles(targets)
+	for _, mod := range goModules {
-	if err != nil {
+		tidy := tc.Go("mod", "tidy", "-diff")
-		log.Fatalf("failed to hash go.mod/go.sum: %v", err)
+		tidy.Dir = mod
-	}
+		build.MustRun(tidy)
-	build.MustRun(new(build.GoToolchain).Go("mod", "tidy"))
-
-	tidied, err := build.HashFiles(targets)
-	if err != nil {
-		log.Fatalf("failed to rehash go.mod/go.sum: %v", err)
-	}
-	if updates := build.DiffHashes(hashes, tidied); len(updates) > 0 {
-		log.Fatalf("files changed on running 'go mod tidy': %v", updates)
 	}
 	fmt.Println("No untidy module files detected.")
 }
@@ -284,38 +310,79 @@ func doTidy() {
 func doGenerate() {
 	var (
 		cachedir = flag.String("cachedir", "./build/cache", "directory for caching binaries.")
+		tc       = new(build.GoToolchain)
 	)
-	// Compute the origin hashes of all the files
-	var hashes map[string][32]byte
 
-	var err error
-	hashes, err = build.HashFolder(".", []string{"tests/testdata", "build/cache"})
-	if err != nil {
-		log.Fatal("Error computing hashes", "err", err)
-	}
 	// Run any go generate steps we might be missing
 	var (
 		protocPath      = downloadProtoc(*cachedir)
 		protocGenGoPath = downloadProtocGenGo(*cachedir)
 	)
-	c := new(build.GoToolchain).Go("generate", "./...")
 	pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")}
-	c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
-	build.MustRun(c)
 
-	// Check if generate file hashes have changed
+	for _, mod := range goModules {
-	generated, err := build.HashFolder(".", []string{"tests/testdata", "build/cache"})
+		// Compute the origin hashes of all the files
-	if err != nil {
+		hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"})
-		log.Fatalf("Error re-computing hashes: %v", err)
+		if err != nil {
-	}
+			log.Fatal("Error computing hashes", "err", err)
-	updates := build.DiffHashes(hashes, generated)
+		}
-	for _, file := range updates {
+
-		log.Printf("File changed: %s", file)
+		c := tc.Go("generate", "./...")
-	}
+		c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
-	if len(updates) != 0 {
+		c.Dir = mod
-		log.Fatal("One or more generated files were updated by running 'go generate ./...'")
+		build.MustRun(c)
+		// Check if generate file hashes have changed
+		generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"})
+		if err != nil {
+			log.Fatalf("Error re-computing hashes: %v", err)
+		}
+		updates := build.DiffHashes(hashes, generated)
+		for _, file := range updates {
+			log.Printf("File changed: %s", file)
+		}
+		if len(updates) != 0 {
+			log.Fatal("One or more generated files were updated by running 'go generate ./...'")
+		}
 	}
 	fmt.Println("No stale files detected.")
+
+	// Run go mod tidy check.
+	for _, mod := range goModules {
+		tidy := tc.Go("mod", "tidy", "-diff")
+		tidy.Dir = mod
+		build.MustRun(tidy)
+	}
+	fmt.Println("No untidy module files detected.")
+}
+
+// doBadDeps verifies whether certain unintended dependencies between some
+// packages leak into the codebase due to a refactor. This is not an exhaustive
+// list, rather something we build up over time at sensitive places.
+func doBadDeps() {
+	baddeps := [][2]string{
+		// Rawdb tends to be a dumping ground for db utils, sometimes leaking the db itself
+		{"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"},
+		{"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/pebbledb"},
+	}
+	tc := new(build.GoToolchain)
+
+	var failed bool
+	for _, rule := range baddeps {
+		out, err := tc.Go("list", "-deps", rule[0]).CombinedOutput()
+		if err != nil {
+			log.Fatalf("Failed to list '%s' dependencies: %v", rule[0], err)
+		}
+		for _, line := range strings.Split(string(out), "\n") {
+			if strings.TrimSpace(line) == rule[1] {
+				log.Printf("Found bad dependency '%s' -> '%s'", rule[0], rule[1])
+				failed = true
+			}
+		}
+	}
+	if failed {
+		log.Fatalf("Bad dependencies detected.")
+	}
+	fmt.Println("No bad dependencies detected.")
 }
 
 // doLint runs golangci-lint on requested packages.
@@ -324,27 +391,42 @@ func doLint(cmdline []string) {
 		cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.")
 	)
 	flag.CommandLine.Parse(cmdline)
-	packages := []string{"./..."}
-	if len(flag.CommandLine.Args()) > 0 {
-		packages = flag.CommandLine.Args()
-	}
 
 	linter := downloadLinter(*cachedir)
-	lflags := []string{"run", "--config", ".golangci.yml"}
+	linter, err := filepath.Abs(linter)
-	build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
+	if err != nil {
+		log.Fatal(err)
+	}
+	config, err := filepath.Abs(".golangci.yml")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	lflags := []string{"run", "--config", config}
+	packages := flag.CommandLine.Args()
+	if len(packages) > 0 {
+		build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
+	} else {
+		// Run for all modules in workspace.
+		for _, mod := range goModules {
+			args := append(lflags, "./...")
+			lintcmd := exec.Command(linter, args...)
+			lintcmd.Dir = mod
+			build.MustRunWithOutput(lintcmd)
+		}
+	}
 	fmt.Println("You have achieved perfection.")
 }
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "golangci")
+	version, err := csdb.FindVersion("golangci")
 	if err != nil {
 		log.Fatal(err)
 	}
 	arch := runtime.GOARCH
 	ext := ".tar.gz"
-
 	if runtime.GOOS == "windows" {
 		ext = ".zip"
 	}
@@ -352,9 +434,8 @@ func downloadLinter(cachedir string) string {
 		arch += "v" + os.Getenv("GOARM")
 	}
 	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
-	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
 	archivePath := filepath.Join(cachedir, base+ext)
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
 		log.Fatal(err)
 	}
 	if err := build.ExtractArchive(archivePath, cachedir); err != nil {
@@ -363,40 +444,6 @@ func downloadLinter(cachedir string) string {
 	return filepath.Join(cachedir, base, "golangci-lint")
 }
 
-// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc
-// in the generate command.  It returns the full path of the directory
-// containing the 'protoc-gen-go' executable.
-func downloadProtocGenGo(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc-gen-go")
-	if err != nil {
-		log.Fatal(err)
-	}
-	baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH)
-	archiveName := baseName
-	if runtime.GOOS == "windows" {
-		archiveName += ".zip"
-	} else {
-		archiveName += ".tar.gz"
-	}
-
-	url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName)
-
-	archivePath := filepath.Join(cachedir, archiveName)
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
-		log.Fatal(err)
-	}
-	extractDest := filepath.Join(cachedir, baseName)
-	if err := build.ExtractArchive(archivePath, extractDest); err != nil {
-		log.Fatal(err)
-	}
-	extractDest, err = filepath.Abs(extractDest)
-	if err != nil {
-		log.Fatal("error resolving absolute path for protoc", "err", err)
-	}
-	return extractDest
-}
-
 // protocArchiveBaseName returns the name of the protoc archive file for
 // the current system, stripped of version and file suffix.
 func protocArchiveBaseName() (string, error) {
@@ -420,12 +467,44 @@ func protocArchiveBaseName() (string, error) {
 	}
 }
 
+// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc
+// in the generate command.  It returns the full path of the directory
+// containing the 'protoc-gen-go' executable.
+func downloadProtocGenGo(cachedir string) string {
+	csdb := download.MustLoadChecksums("build/checksums.txt")
+	version, err := csdb.FindVersion("protoc-gen-go")
+	if err != nil {
+		log.Fatal(err)
+	}
+	baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH)
+	archiveName := baseName
+	if runtime.GOOS == "windows" {
+		archiveName += ".zip"
+	} else {
+		archiveName += ".tar.gz"
+	}
+
+	archivePath := path.Join(cachedir, archiveName)
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
+		log.Fatal(err)
+	}
+	extractDest := filepath.Join(cachedir, baseName)
+	if err := build.ExtractArchive(archivePath, extractDest); err != nil {
+		log.Fatal(err)
+	}
+	extractDest, err = filepath.Abs(extractDest)
+	if err != nil {
+		log.Fatal("error resolving absolute path for protoc", "err", err)
+	}
+	return extractDest
+}
+
 // downloadProtoc downloads the prebuilt protoc binary used to lint generated
 // files as a CI step.  It returns the full path to the directory containing
 // the protoc executable.
 func downloadProtoc(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc")
+	version, err := csdb.FindVersion("protoc")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -436,10 +515,8 @@ func downloadProtoc(cachedir string) string {
 
 	fileName := fmt.Sprintf("protoc-%s-%s", version, baseName)
 	archiveFileName := fileName + ".zip"
-	url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName)
 	archivePath := filepath.Join(cachedir, archiveFileName)
-
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
 		log.Fatal(err)
 	}
 	extractDest := filepath.Join(cachedir, fileName)
@@ -462,11 +539,12 @@ func doXgo(cmdline []string) {
 	env := build.Env()
 
 	// Make sure xgo is available for cross compilation
-	gogetxgo := goTool("get", "github.com/karalabe/xgo")
+	tc := build.GoToolchain{}
+	gogetxgo := tc.Go("get", "github.com/karalabe/xgo")
 	build.MustRun(gogetxgo)
 
 	// If all tools building is requested, build everything the builder wants
-	args := append(buildFlags(env), flag.Args()...)
+	args := append(buildFlags(env, false, nil), flag.Args()...)
 
 	if *alltools {
 		args = append(args, []string{"--dest", GOBIN}...)

internal/build/download.go

@@ -1,151 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package build
-
-import (
-	"bufio"
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// ChecksumDB keeps file checksums.
-type ChecksumDB struct {
-	allChecksums []string
-}
-
-// MustLoadChecksums loads a file containing checksums.
-func MustLoadChecksums(file string) *ChecksumDB {
-	content, err := os.ReadFile(file)
-	if err != nil {
-		log.Fatal("can't load checksum file: " + err.Error())
-	}
-	return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
-}
-
-// Verify checks whether the given file is valid according to the checksum database.
-func (db *ChecksumDB) Verify(path string) error {
-	fd, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	defer fd.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
-		return err
-	}
-	fileHash := hex.EncodeToString(h.Sum(nil))
-	if !db.findHash(filepath.Base(path), fileHash) {
-		return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
-	}
-	return nil
-}
-
-func (db *ChecksumDB) findHash(basename, hash string) bool {
-	want := hash + "  " + basename
-	for _, line := range db.allChecksums {
-		if strings.TrimSpace(line) == want {
-			return true
-		}
-	}
-	return false
-}
-
-// DownloadFile downloads a file and verifies its checksum.
-func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
-	if err := db.Verify(dstPath); err == nil {
-		fmt.Printf("%s is up-to-date\n", dstPath)
-		return nil
-	}
-	fmt.Printf("%s is stale\n", dstPath)
-	fmt.Printf("downloading from %s\n", url)
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return fmt.Errorf("download error: %v", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("download error: status %d", resp.StatusCode)
-	}
-	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
-		return err
-	}
-	fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
-	if err != nil {
-		return err
-	}
-	dst := newDownloadWriter(fd, resp.ContentLength)
-	_, err = io.Copy(dst, resp.Body)
-	dst.Close()
-	if err != nil {
-		return err
-	}
-	return db.Verify(dstPath)
-}
-
-type downloadWriter struct {
-	file    *os.File
-	dstBuf  *bufio.Writer
-	size    int64
-	written int64
-	lastpct int64
-}
-
-func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
-	return &downloadWriter{
-		file:   dst,
-		dstBuf: bufio.NewWriter(dst),
-		size:   size,
-	}
-}
-
-func (w *downloadWriter) Write(buf []byte) (int, error) {
-	n, err := w.dstBuf.Write(buf)
-
-	// Report progress.
-	w.written += int64(n)
-	pct := w.written * 10 / w.size * 10
-	if pct != w.lastpct {
-		if w.lastpct != 0 {
-			fmt.Print("...")
-		}
-		fmt.Print(pct, "%")
-		w.lastpct = pct
-	}
-	return n, err
-}
-
-func (w *downloadWriter) Close() error {
-	if w.lastpct > 0 {
-		fmt.Println() // Finish the progress line.
-	}
-	flushErr := w.dstBuf.Flush()
-	closeErr := w.file.Close()
-	if flushErr != nil {
-		return flushErr
-	}
-	return closeErr
-}

internal/build/file.go

@@ -25,32 +25,6 @@ import (
 	"strings"
 )
 
-// FileExist checks if a file exists at path.
-func FileExist(path string) bool {
-	_, err := os.Stat(path)
-	if err != nil && os.IsNotExist(err) {
-		return false
-	}
-	return true
-}
-
-// HashFiles iterates the provided set of files, computing the hash of each.
-func HashFiles(files []string) (map[string][32]byte, error) {
-	res := make(map[string][32]byte)
-	for _, filePath := range files {
-		f, err := os.OpenFile(filePath, os.O_RDONLY, 0666)
-		if err != nil {
-			return nil, err
-		}
-		hasher := sha256.New()
-		if _, err := io.Copy(hasher, f); err != nil {
-			return nil, err
-		}
-		res[filePath] = [32]byte(hasher.Sum(nil))
-	}
-	return res, nil
-}
-
 // HashFolder iterates all files under the given directory, computing the hash
 // of each.
 func HashFolder(folder string, exlude []string) (map[string][32]byte, error) {

internal/build/gotool.go

@@ -24,6 +24,8 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"github.com/XinFinOrg/XDPoSChain/internal/download"
 )
 
 type GoToolchain struct {
@@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
 
 // DownloadGo downloads the Go binary distribution and unpacks it into a temporary
 // directory. It returns the GOROOT of the unpacked toolchain.
-func DownloadGo(csdb *ChecksumDB) string {
+func DownloadGo(csdb *download.ChecksumDB) string {
-	version, err := Version(csdb, "golang")
+	version, err := csdb.FindVersion("golang")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
 	}
 	return goroot
 }
-
-// Version returns the versions defined in the checksumdb.
-func Version(csdb *ChecksumDB, version string) (string, error) {
-	for _, l := range csdb.allChecksums {
-		if !strings.HasPrefix(l, "# version:") {
-			continue
-		}
-		v := strings.Split(l, ":")[1]
-		parts := strings.Split(v, " ")
-		if len(parts) != 2 {
-			log.Print("Erroneous version-string", "v", l)
-			continue
-		}
-		if parts[0] == version {
-			return parts[1], nil
-		}
-	}
-	return "", fmt.Errorf("no version found for '%v'", version)
-}
-
-// DownloadAndVerifyChecksums downloads all files and checks that they match
-// the checksum given in checksums.txt.
-// This task can be used to sanity-check new checksums.
-func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
-	var (
-		base   = ""
-		ucache = os.TempDir()
-	)
-	for _, l := range csdb.allChecksums {
-		if strings.HasPrefix(l, "# https://") {
-			base = l[2:]
-			continue
-		}
-		if strings.HasPrefix(l, "#") {
-			continue
-		}
-		hashFile := strings.Split(l, "  ")
-		if len(hashFile) != 2 {
-			continue
-		}
-		file := hashFile[1]
-		url := base + file
-		dst := filepath.Join(ucache, file)
-		if err := csdb.DownloadFile(url, dst); err != nil {
-			log.Print(err)
-		}
-	}
-}

internal/build/util.go

@@ -17,16 +17,18 @@
 package build
 
 import (
+	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 )
 
@@ -35,6 +37,9 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
 // MustRun executes the given command and exits the host process for
 // any error.
 func MustRun(cmd *exec.Cmd) {
+	if cmd.Dir != "" && cmd.Dir != "." {
+		fmt.Printf("(in %s) ", cmd.Dir)
+	}
 	fmt.Println(">>>", printArgs(cmd.Args))
 	if !*DryRunFlag {
 		cmd.Stderr = os.Stderr
@@ -67,6 +72,13 @@ func MustRunCommand(cmd string, args ...string) {
 // printed while it runs. This is useful for CI builds where the process will be stopped
 // when there is no output.
 func MustRunCommandWithOutput(cmd string, args ...string) {
+	MustRunWithOutput(exec.Command(cmd, args...))
+}
+
+// MustRunWithOutput runs the given command, and ensures that some output will be printed
+// while it runs. This is useful for CI builds where the process will be stopped when
+// there is no output.
+func MustRunWithOutput(cmd *exec.Cmd) {
 	interval := time.NewTicker(time.Minute)
 	done := make(chan struct{})
 	defer interval.Stop()
@@ -81,16 +93,7 @@ func MustRunCommandWithOutput(cmd string, args ...string) {
 			}
 		}
 	}()
-	MustRun(exec.Command(cmd, args...))
+	MustRun(cmd)
-}
-
-// GOPATH returns the value that the GOPATH environment
-// variable should be set to.
-func GOPATH() string {
-	if os.Getenv("GOPATH") == "" {
-		log.Fatal("GOPATH is not set")
-	}
-	return os.Getenv("GOPATH")
 }
 
 var warnedAboutGit bool
@@ -101,13 +104,14 @@ func RunGit(args ...string) string {
 	cmd := exec.Command("git", args...)
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout, cmd.Stderr = &stdout, &stderr
-	if err := cmd.Run(); err == exec.ErrNotFound {
+	if err := cmd.Run(); err != nil {
-		if !warnedAboutGit {
+		if e, ok := err.(*exec.Error); ok && e.Err == exec.ErrNotFound {
-			log.Println("Warning: can't find 'git' in PATH")
+			if !warnedAboutGit {
-			warnedAboutGit = true
+				log.Println("Warning: can't find 'git' in PATH")
+				warnedAboutGit = true
+			}
+			return ""
 		}
-		return ""
-	} else if err != nil {
 		log.Fatal(strings.Join(cmd.Args, " "), ": ", err, "\n", stderr.String())
 	}
 	return strings.TrimSpace(stdout.String())
@@ -122,46 +126,136 @@ func readGitFile(file string) string {
 	return strings.TrimSpace(string(content))
 }
 
-// GoTool returns the command that runs a go tool. This uses go from GOROOT instead of PATH
+// Render renders the given template file into outputFile.
-// so that go commands executed by build use the same version of Go as the 'host' that runs
+func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) {
-// build code. e.g.
+	tpl := template.Must(template.ParseFiles(templateFile))
-//
+	render(tpl, outputFile, outputPerm, x)
-//	/usr/lib/go-1.11/bin/go run build/ci.go ...
-//
-// runs using go 1.11 and invokes go 1.11 tools from the same GOROOT. This is also important
-// because runtime.Version checks on the host should match the tools that are run.
-func GoTool(tool string, args ...string) *exec.Cmd {
-	args = append([]string{tool}, args...)
-	return exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
 }
 
-// ExpandPackages expands a cmd/go import path pattern, skip vendor
+// RenderString renders the given template string into outputFile.
-// packages, and skip time-consuming tests according to quick flag.
+func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) {
-func ExpandPackages(patterns []string, quick bool) []string {
+	tpl := template.Must(template.New("").Parse(templateContent))
-	expand := false
+	render(tpl, outputFile, outputPerm, x)
-	for _, pkg := range patterns {
+}
-		if strings.Contains(pkg, "...") {
+
-			expand = true
+func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x interface{}) {
-		}
+	if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil {
-	}
+		log.Fatal(err)
-	if expand {
+	}
-		cmd := GoTool("list", patterns...)
+	out, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, outputPerm)
-		out, err := cmd.CombinedOutput()
+	if err != nil {
-		if err != nil {
+		log.Fatal(err)
-			log.Fatalf("package listing failed: %v\n%s", err, string(out))
+	}
-		}
+	if err := tpl.Execute(out, x); err != nil {
-		var packages []string
+		log.Fatal(err)
-		for _, line := range strings.Split(string(out), "\n") {
+	}
-			if strings.Contains(line, "/vendor/") {
+	if err := out.Close(); err != nil {
-				continue
+		log.Fatal(err)
-			}
+	}
-			if quick && strings.Contains(line, "/consensus/tests/engine_v2_tests") {
+}
-				continue
+
-			}
+// UploadSFTP uploads files to a remote host using the sftp command line tool.
-			packages = append(packages, strings.TrimSpace(line))
+// The destination host may be specified either as [user@]host: or as a URI in
-
+// the form sftp://[user@]host[:port].
-		}
+func UploadSFTP(identityFile, host, dir string, files []string) error {
-		return packages
+	sftp := exec.Command("sftp")
-	}
+	sftp.Stderr = os.Stderr
-	return patterns
+	if identityFile != "" {
+		sftp.Args = append(sftp.Args, "-i", identityFile)
+	}
+	sftp.Args = append(sftp.Args, host)
+	fmt.Println(">>>", printArgs(sftp.Args))
+	if *DryRunFlag {
+		return nil
+	}
+
+	stdin, err := sftp.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("can't create stdin pipe for sftp: %v", err)
+	}
+	stdout, err := sftp.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("can't create stdout pipe for sftp: %v", err)
+	}
+	if err := sftp.Start(); err != nil {
+		return err
+	}
+	in := io.MultiWriter(stdin, os.Stdout)
+	for _, f := range files {
+		fmt.Fprintln(in, "put", f, filepath.Join(dir, filepath.Base(f)))
+	}
+	fmt.Fprintln(in, "exit")
+	// Some issue with the PPA sftp server makes it so the server does not
+	// respond properly to a 'bye', 'exit' or 'quit' from the client.
+	// To work around that, we check the output, and when we see the client
+	// exit command, we do a hard exit.
+	// See
+	// https://github.com/kolban-google/sftp-gcs/issues/23
+	// https://github.com/mscdex/ssh2/pull/1111
+	aborted := false
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			txt := scanner.Text()
+			fmt.Println(txt)
+			if txt == "sftp> exit" {
+				// Give it .5 seconds to exit (server might be fixed), then
+				// hard kill it from the outside
+				time.Sleep(500 * time.Millisecond)
+				aborted = true
+				sftp.Process.Kill()
+			}
+		}
+	}()
+	stdin.Close()
+	err = sftp.Wait()
+	if aborted {
+		return nil
+	}
+	return err
+}
+
+// FindMainPackages finds all 'main' packages in the given directory and returns their
+// package paths.
+func FindMainPackages(tc *GoToolchain, pattern string) []string {
+	list := tc.Go("list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, pattern)
+	output, err := list.Output()
+	if err != nil {
+		log.Fatal("go list failed:", err)
+	}
+	var result []string
+	for l := range bytes.Lines(output) {
+		l = bytes.TrimSpace(l)
+		if len(l) > 0 {
+			result = append(result, string(l))
+		}
+	}
+	return result
+}
+
+// FindAllPackages expands a cmd/go import path pattern.
+func FindAllPackages(tc *GoToolchain) []string {
+	list := tc.Go("list", "./...")
+	output, err := list.Output()
+	if err != nil {
+		log.Fatal("go list failed:", err)
+	}
+	result := make([]string, 0, len(output))
+	for line := range bytes.Lines(output) {
+		pkg := bytes.TrimSpace(line)
+		if len(pkg) > 0 {
+			result = append(result, string(pkg))
+		}
+	}
+	return result
+}
+
+// GOPATH returns the value that the GOPATH environment
+// variable should be set to.
+func GOPATH() string {
+	gopath := os.Getenv("GOPATH")
+	if gopath == "" {
+		log.Fatal("GOPATH is not set")
+	}
+	return gopath
 }

internal/download/download.go

@@ -0,0 +1,298 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package download implements checksum-verified file downloads.
+package download
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"iter"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ChecksumDB keeps file checksums and tool versions.
+type ChecksumDB struct {
+	hashes   []hashEntry
+	versions []versionEntry
+}
+
+type versionEntry struct {
+	name    string
+	version string
+}
+
+type hashEntry struct {
+	hash string
+	file string
+	url  *url.URL
+}
+
+// MustLoadChecksums loads a file containing checksums.
+func MustLoadChecksums(file string) *ChecksumDB {
+	content, err := os.ReadFile(file)
+	if err != nil {
+		panic("can't load checksum file: " + err.Error())
+	}
+	db, err := ParseChecksums(content)
+	if err != nil {
+		panic(fmt.Sprintf("invalid checksums in %s: %v", file, err))
+	}
+	return db
+}
+
+// ParseChecksums parses a checksum database.
+func ParseChecksums(input []byte) (*ChecksumDB, error) {
+	var (
+		csdb    = new(ChecksumDB)
+		rd      = bytes.NewBuffer(input)
+		lastURL *url.URL
+	)
+	for lineNum := 1; ; lineNum++ {
+		line, err := rd.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		line = strings.TrimSpace(line)
+		switch {
+		case line == "":
+			// Blank lines are allowed, and they reset the current urlEntry.
+			lastURL = nil
+
+		case strings.HasPrefix(line, "#"):
+			// It's a comment. Some comments have special meaning.
+			content := strings.TrimLeft(line, "# ")
+			switch {
+			case strings.HasPrefix(content, "version:"):
+				// Version comments define the version of a tool.
+				v := strings.Split(content, ":")[1]
+				parts := strings.Split(v, " ")
+				if len(parts) != 2 {
+					return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
+				}
+				csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
+
+			case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
+				// URL comments define the URL where the following files are found. Here
+				// we keep track of the last found urlEntry and attach it to each file later.
+				u, err := url.Parse(content)
+				if err != nil {
+					return nil, fmt.Errorf("line %d: invalid URL: %v", lineNum, err)
+				}
+				lastURL = u
+			}
+
+		default:
+			// It's a file hash entry.
+			fields := strings.Fields(line)
+			if len(fields) != 2 {
+				return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
+			}
+			csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
+		}
+	}
+	return csdb, nil
+}
+
+// Files returns an iterator over all file names.
+func (db *ChecksumDB) Files() iter.Seq[string] {
+	return func(yield func(string) bool) {
+		for _, e := range db.hashes {
+			if !yield(e.file) {
+				return
+			}
+		}
+	}
+}
+
+// DownloadAndVerifyAll downloads all files and checks that they match the checksum given in
+// the database. This task can be used to sanity-check new checksums.
+func (db *ChecksumDB) DownloadAndVerifyAll() {
+	var tmp = os.TempDir()
+	for _, e := range db.hashes {
+		if e.url == nil {
+			fmt.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
+			continue
+		}
+		url := e.url.JoinPath(e.file).String()
+		dst := filepath.Join(tmp, e.file)
+		if err := db.DownloadFile(url, dst); err != nil {
+			fmt.Println("error:", err)
+		}
+	}
+}
+
+// verifyHash checks that the file at 'path' has the expected hash.
+func verifyHash(path, expectedHash string) error {
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
+		return err
+	}
+	fileHash := hex.EncodeToString(h.Sum(nil))
+	if fileHash != expectedHash {
+		return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
+	}
+	return nil
+}
+
+// DownloadFileFromKnownURL downloads a file from the URL defined in the checksum database.
+func (db *ChecksumDB) DownloadFileFromKnownURL(dstPath string) error {
+	base := filepath.Base(dstPath)
+	url, err := db.FindURL(base)
+	if err != nil {
+		return err
+	}
+	return db.DownloadFile(url, dstPath)
+}
+
+// DownloadFile downloads a file and verifies its checksum.
+func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
+	basename := filepath.Base(dstPath)
+	hash := db.findHash(basename)
+	if hash == "" {
+		return fmt.Errorf("no known hash for file %q", basename)
+	}
+	// Shortcut if already downloaded.
+	if verifyHash(dstPath, hash) == nil {
+		fmt.Printf("%s is up-to-date\n", dstPath)
+		return nil
+	}
+
+	fmt.Printf("%s is stale\n", dstPath)
+	fmt.Printf("downloading from %s\n", url)
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("download error: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("download error: status %d", resp.StatusCode)
+	}
+	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+		return err
+	}
+
+	// Download to a temporary file.
+	tmpfile := dstPath + ".tmp"
+	fd, err := os.OpenFile(tmpfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	dst := newDownloadWriter(fd, resp.ContentLength)
+	_, err = io.Copy(dst, resp.Body)
+	dst.Close()
+	if err != nil {
+		os.Remove(tmpfile)
+		return err
+	}
+	if err := verifyHash(tmpfile, hash); err != nil {
+		os.Remove(tmpfile)
+		return err
+	}
+	// It's valid, rename to dstPath to complete the download.
+	return os.Rename(tmpfile, dstPath)
+}
+
+// findHash returns the known hash of a file.
+func (db *ChecksumDB) findHash(basename string) string {
+	for _, e := range db.hashes {
+		if e.file == basename {
+			return e.hash
+		}
+	}
+	return ""
+}
+
+// FindVersion returns the current known version of a tool, if it is defined in the file.
+func (db *ChecksumDB) FindVersion(tool string) (string, error) {
+	for _, e := range db.versions {
+		if e.name == tool {
+			return e.version, nil
+		}
+	}
+	return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
+}
+
+// FindURL gets the URL for a file.
+func (db *ChecksumDB) FindURL(basename string) (string, error) {
+	for _, e := range db.hashes {
+		if e.file == basename {
+			if e.url == nil {
+				return "", fmt.Errorf("file %q has no URL defined", e.file)
+			}
+			return e.url.JoinPath(e.file).String(), nil
+		}
+	}
+	return "", fmt.Errorf("file %q does not exist in checksum database", basename)
+}
+
+type downloadWriter struct {
+	file    *os.File
+	dstBuf  *bufio.Writer
+	size    int64
+	written int64
+	lastpct int64
+}
+
+func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
+	return &downloadWriter{
+		file:   dst,
+		dstBuf: bufio.NewWriter(dst),
+		size:   size,
+	}
+}
+
+func (w *downloadWriter) Write(buf []byte) (int, error) {
+	n, err := w.dstBuf.Write(buf)
+
+	// Report progress.
+	w.written += int64(n)
+	pct := w.written * 10 / w.size * 10
+	if pct != w.lastpct {
+		if w.lastpct != 0 {
+			fmt.Print("...")
+		}
+		fmt.Print(pct, "%")
+		w.lastpct = pct
+	}
+	return n, err
+}
+
+func (w *downloadWriter) Close() error {
+	if w.lastpct > 0 {
+		fmt.Println() // Finish the progress line.
+	}
+	flushErr := w.dstBuf.Flush()
+	closeErr := w.file.Close()
+	if flushErr != nil {
+		return flushErr
+	}
+	return closeErr
+}