mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-19 21:31:37 +00:00
build, internal, Makefile: upgrade package build (#1746)
This commit is contained in:
Daniel Liu
GitHub
parent
c53b38e5fe
commit
25f07b1040
7 changed files with 743 additions and 492 deletions
10
Makefile
10
Makefile
|
|
@ -45,7 +45,7 @@ all:
|
|||
test: all
|
||||
go run build/ci.go test -failfast
|
||||
|
||||
#? quick-test: Run the tests except time-consuming tests.
|
||||
#? quick-test: Run the tests except time-consuming packages.
|
||||
quick-test: all
|
||||
go run build/ci.go test --quick -failfast
|
||||
|
||||
|
|
@ -53,14 +53,18 @@ quick-test: all
|
|||
lint: ## Run linters.
|
||||
$(GORUN) build/ci.go lint
|
||||
|
||||
#? tidy: Verify go.mod and go.sum by 'go mod tidy'
|
||||
#? tidy: Verify go.mod and go.sum are updated.
|
||||
tidy: ## Run 'go mod tidy'.
|
||||
$(GORUN) build/ci.go tidy
|
||||
|
||||
#? generate: Verify everything is 'go generate'-ed
|
||||
#? generate: Verify everything is 'go generate'.
|
||||
generate: ## Run 'go generate ./...'.
|
||||
$(GORUN) build/ci.go generate
|
||||
|
||||
#? baddeps: Verify certain dependencies are avoided.
|
||||
baddeps:
|
||||
$(GORUN) build/ci.go baddeps
|
||||
|
||||
#? fmt: Ensure consistent code formatting.
|
||||
fmt:
|
||||
gofmt -s -w $(shell find . -name "*.go")
|
||||
|
|
|
|||
488
build/ci.go
488
build/ci.go
|
|
@ -24,14 +24,15 @@ Usage: go run build/ci.go <command> <command flags/arguments>
|
|||
|
||||
Available commands are:
|
||||
|
||||
lint -- runs certain pre-selected linters
|
||||
tidy -- verifies that everything is 'go mod tidy'-ed
|
||||
generate -- verifies that everything is 'go generate'-ed
|
||||
lint -- runs certain pre-selected linters
|
||||
tidy -- verifies that everything is 'go mod tidy'-ed
|
||||
generate -- verifies that everything is 'go generate'-ed
|
||||
baddeps -- verifies that certain dependencies are avoided
|
||||
|
||||
install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
|
||||
test [ -coverage ] [ packages... ] -- runs the tests
|
||||
importkeys -- imports signing keys from env
|
||||
xgo [ -alltools ] [ options ] -- cross builds according to options
|
||||
install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables
|
||||
test [ -coverage ] [ packages... ] -- runs the tests
|
||||
importkeys -- imports signing keys from env
|
||||
xgo [ -alltools ] [ options ] -- cross builds according to options
|
||||
|
||||
For all commands, -n prevents execution of external programs (dry run mode).
|
||||
*/
|
||||
|
|
@ -40,31 +41,36 @@ package main
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/XinFinOrg/XDPoSChain/common"
|
||||
"github.com/XinFinOrg/XDPoSChain/internal/build"
|
||||
"github.com/XinFinOrg/XDPoSChain/internal/download"
|
||||
)
|
||||
|
||||
var (
|
||||
goModules = []string{
|
||||
".",
|
||||
}
|
||||
|
||||
// Files that end up in the geth-alltools*.zip archive.
|
||||
allToolsArchiveFiles = []string{
|
||||
"COPYING",
|
||||
executablePath("abigen"),
|
||||
executablePath("bootnode"),
|
||||
executablePath("ethkey"),
|
||||
executablePath("evm"),
|
||||
executablePath("geth"),
|
||||
executablePath("p2psim"),
|
||||
executablePath("puppeth"),
|
||||
executablePath("rlpdump"),
|
||||
executablePath("swarm"),
|
||||
executablePath("wnode"),
|
||||
executablePath("XDC"),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -97,6 +103,8 @@ func main() {
|
|||
doTidy()
|
||||
case "generate":
|
||||
doGenerate()
|
||||
case "baddeps":
|
||||
doBadDeps()
|
||||
case "xgo":
|
||||
doXgo(os.Args[2:])
|
||||
default:
|
||||
|
|
@ -108,145 +116,131 @@ func main() {
|
|||
|
||||
func doInstall(cmdline []string) {
|
||||
var (
|
||||
arch = flag.String("arch", "", "Architecture to cross build for")
|
||||
cc = flag.String("cc", "", "C compiler to cross build with")
|
||||
dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
|
||||
arch = flag.String("arch", "", "Architecture to cross build for")
|
||||
cc = flag.String("cc", "", "C compiler to cross build with")
|
||||
staticlink = flag.Bool("static", false, "Create statically-linked executable")
|
||||
)
|
||||
flag.CommandLine.Parse(cmdline)
|
||||
env := build.Env()
|
||||
|
||||
// Check Go version. People regularly open issues about compilation
|
||||
// failure with outdated Go. This should save them the trouble.
|
||||
if !strings.Contains(runtime.Version(), "devel") {
|
||||
// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
|
||||
var minor int
|
||||
fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
|
||||
// Configure the toolchain.
|
||||
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
|
||||
if *dlgo {
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
tc.Root = build.DownloadGo(csdb)
|
||||
}
|
||||
// Disable CLI markdown doc generation in release builds.
|
||||
buildTags := []string{"urfave_cli_no_docs"}
|
||||
|
||||
if minor < 25 {
|
||||
log.Println("You have Go version", runtime.Version())
|
||||
log.Println("XDC requires at least Go version 1.25 and cannot")
|
||||
log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
// Compile packages given as arguments, or everything if there are no arguments.
|
||||
packages := []string{"./..."}
|
||||
if flag.NArg() > 0 {
|
||||
packages = flag.Args()
|
||||
}
|
||||
// packages = build.ExpandPackagesNoVendor(packages)
|
||||
// Configure the build.
|
||||
gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...)
|
||||
|
||||
if *arch == "" || *arch == runtime.GOARCH {
|
||||
goinstall := goTool("install", buildFlags(env)...)
|
||||
goinstall.Args = append(goinstall.Args, "-v")
|
||||
goinstall.Args = append(goinstall.Args, packages...)
|
||||
build.MustRun(goinstall)
|
||||
return
|
||||
}
|
||||
// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
|
||||
if *arch == "arm" {
|
||||
os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
|
||||
for _, path := range filepath.SplitList(build.GOPATH()) {
|
||||
os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm"))
|
||||
}
|
||||
}
|
||||
// Seems we are cross compiling, work around forbidden GOBIN
|
||||
goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
|
||||
goinstall.Args = append(goinstall.Args, "-v")
|
||||
goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
|
||||
goinstall.Args = append(goinstall.Args, packages...)
|
||||
build.MustRun(goinstall)
|
||||
// Show packages during build.
|
||||
gobuild.Args = append(gobuild.Args, "-v")
|
||||
|
||||
if cmds, err := os.ReadDir("cmd"); err == nil {
|
||||
for _, cmd := range cmds {
|
||||
pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for name := range pkgs {
|
||||
if name == "main" {
|
||||
gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
|
||||
gobuild.Args = append(gobuild.Args, "-v")
|
||||
gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
|
||||
gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
|
||||
build.MustRun(gobuild)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Now we choose what we're even building.
|
||||
// Default: collect all 'main' packages in cmd/ and build those.
|
||||
packages := flag.Args()
|
||||
if len(packages) == 0 {
|
||||
// NOTE: to collect all main packages, use:
|
||||
// packages = build.FindMainPackages(&tc, "./...")
|
||||
packages = build.FindMainPackages(&tc, "./cmd/...")
|
||||
}
|
||||
|
||||
// Do the build!
|
||||
for _, pkg := range packages {
|
||||
args := slices.Clone(gobuild.Args)
|
||||
args = append(args, "-o", executablePath(path.Base(pkg)))
|
||||
args = append(args, pkg)
|
||||
build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env})
|
||||
}
|
||||
}
|
||||
|
||||
func buildFlags(env build.Environment) (flags []string) {
|
||||
// buildFlags returns the go tool flags for building.
|
||||
func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) {
|
||||
var ld []string
|
||||
// See https://github.com/golang/go/issues/33772#issuecomment-528176001
|
||||
// We need to set --buildid to the linker here, and also pass --build-id to the
|
||||
// cgo-linker further down.
|
||||
ld = append(ld, "--buildid=none")
|
||||
if env.Commit != "" {
|
||||
ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitCommit="+env.Commit)
|
||||
ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitDate="+env.Date)
|
||||
}
|
||||
// Strip DWARF on darwin. This used to be required for certain things,
|
||||
// and there is no downside to this, so we just keep doing it.
|
||||
if runtime.GOOS == "darwin" {
|
||||
ld = append(ld, "-s")
|
||||
}
|
||||
|
||||
if runtime.GOOS == "linux" {
|
||||
// Enforce the stacksize to 8M, which is the case on most platforms apart from
|
||||
// alpine Linux.
|
||||
// See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options
|
||||
// regarding the options --build-id=none and --strip-all. It is needed for
|
||||
// reproducible builds; removing references to temporary files in C-land, and
|
||||
// making build-id reproducibly absent.
|
||||
extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"}
|
||||
if staticLinking {
|
||||
extld = append(extld, "-static")
|
||||
// Under static linking, use of certain glibc features must be
|
||||
// disabled to avoid shared library dependencies.
|
||||
buildTags = append(buildTags, "osusergo", "netgo")
|
||||
}
|
||||
ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'")
|
||||
}
|
||||
if len(ld) > 0 {
|
||||
flags = append(flags, "-ldflags", strings.Join(ld, " "))
|
||||
}
|
||||
if len(buildTags) > 0 {
|
||||
flags = append(flags, "-tags", strings.Join(buildTags, ","))
|
||||
}
|
||||
// We use -trimpath to avoid leaking local paths into the built executables.
|
||||
flags = append(flags, "-trimpath")
|
||||
return flags
|
||||
}
|
||||
|
||||
func goTool(subcmd string, args ...string) *exec.Cmd {
|
||||
return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
|
||||
}
|
||||
|
||||
func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
|
||||
cmd := build.GoTool(subcmd, args...)
|
||||
cmd.Env = []string{"GOPATH=" + build.GOPATH()}
|
||||
if arch == "" || arch == runtime.GOARCH {
|
||||
cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
|
||||
} else {
|
||||
cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
|
||||
cmd.Env = append(cmd.Env, "GOARCH="+arch)
|
||||
}
|
||||
if cc != "" {
|
||||
cmd.Env = append(cmd.Env, "CC="+cc)
|
||||
}
|
||||
for _, e := range os.Environ() {
|
||||
if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
|
||||
continue
|
||||
}
|
||||
cmd.Env = append(cmd.Env, e)
|
||||
}
|
||||
return cmd
|
||||
}
|
||||
|
||||
// Running The Tests
|
||||
//
|
||||
// "tests" also includes static analysis tools such as vet.
|
||||
func doTest(cmdline []string) {
|
||||
coverage := flag.Bool("coverage", false, "Whether to record code coverage")
|
||||
verbose := flag.Bool("v", false, "Whether to log verbosely")
|
||||
quick := flag.Bool("quick", false, "Whether to skip long time test")
|
||||
failfast := flag.Bool("failfast", false, "Do not start new tests after the first test failure")
|
||||
flag.CommandLine.Parse(cmdline)
|
||||
env := build.Env()
|
||||
|
||||
packages := []string{"./..."} // if a package has no test files, the lines in that package are not added to the total line count
|
||||
if len(flag.CommandLine.Args()) > 0 {
|
||||
packages = flag.CommandLine.Args()
|
||||
} else {
|
||||
// added all files in all packages (except vendor) to coverage report files count, even there is no test file in the package
|
||||
packages = build.ExpandPackages(packages, *quick)
|
||||
func doTest(cmdline []string) {
|
||||
var (
|
||||
dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
|
||||
arch = flag.String("arch", "", "Run tests for given architecture")
|
||||
cc = flag.String("cc", "", "Sets C compiler binary")
|
||||
coverage = flag.Bool("coverage", false, "Whether to record code coverage")
|
||||
verbose = flag.Bool("v", false, "Whether to log verbosely")
|
||||
race = flag.Bool("race", false, "Execute the race detector")
|
||||
short = flag.Bool("short", false, "Pass the 'short'-flag to go test")
|
||||
threads = flag.Int("p", 1, "Number of CPU threads to use for testing")
|
||||
quick = flag.Bool("quick", false, "Whether to skip long time test")
|
||||
failfast = flag.Bool("failfast", false, "Do not start new tests after the first test failure")
|
||||
)
|
||||
flag.CommandLine.Parse(cmdline)
|
||||
|
||||
// Load checksums file (needed for both spec tests and dlgo)
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
|
||||
// Configure the toolchain.
|
||||
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
|
||||
if *dlgo {
|
||||
tc.Root = build.DownloadGo(csdb)
|
||||
}
|
||||
|
||||
// Run analysis tools before the tests.
|
||||
// build.MustRun(goTool("vet", packages...))
|
||||
gotest := tc.Go("test")
|
||||
|
||||
// CI needs a bit more time for the statetests (default 45m).
|
||||
gotest.Args = append(gotest.Args, "-timeout=45m")
|
||||
|
||||
// Enable integration-tests
|
||||
gotest.Args = append(gotest.Args, "-tags=integrationtests")
|
||||
|
||||
// Run the actual tests.
|
||||
// gotest := goTool("test", buildFlags(env)...)
|
||||
// Test a single package at a time. CI builders are slow
|
||||
// and some tests run into timeouts under load.
|
||||
gotest := goTool("test", buildFlags(env)...)
|
||||
gotest.Args = append(gotest.Args, "-p", "1")
|
||||
gotest.Args = append(gotest.Args, "-p", fmt.Sprintf("%d", *threads))
|
||||
if *coverage {
|
||||
gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover", "-coverprofile=coverage.txt")
|
||||
gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
|
||||
}
|
||||
if *verbose {
|
||||
gotest.Args = append(gotest.Args, "-v")
|
||||
|
|
@ -254,27 +248,59 @@ func doTest(cmdline []string) {
|
|||
if *failfast {
|
||||
gotest.Args = append(gotest.Args, "-failfast")
|
||||
}
|
||||
if *race {
|
||||
gotest.Args = append(gotest.Args, "-race")
|
||||
}
|
||||
if *short {
|
||||
gotest.Args = append(gotest.Args, "-short")
|
||||
}
|
||||
|
||||
packages := flag.CommandLine.Args()
|
||||
if len(packages) > 0 {
|
||||
if *quick {
|
||||
packages = filterPackages(packages)
|
||||
}
|
||||
gotest.Args = append(gotest.Args, packages...)
|
||||
build.MustRun(gotest)
|
||||
return
|
||||
}
|
||||
|
||||
// No packages specified, run all tests for all modules.
|
||||
if *quick {
|
||||
packages = filterPackages(build.FindAllPackages(&tc))
|
||||
} else {
|
||||
packages = []string{"./..."}
|
||||
}
|
||||
gotest.Args = append(gotest.Args, packages...)
|
||||
build.MustRun(gotest)
|
||||
for _, mod := range goModules {
|
||||
test := *gotest
|
||||
test.Dir = mod
|
||||
build.MustRun(&test)
|
||||
}
|
||||
}
|
||||
|
||||
// doTidy assets that the Go modules files are tidied already.
|
||||
// filterPackages removes time-consuming packages.
|
||||
func filterPackages(packages []string) []string {
|
||||
var filtered []string
|
||||
|
||||
for _, pkg := range packages {
|
||||
if strings.Contains(pkg, "/consensus/tests/engine_v2_tests") {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, pkg)
|
||||
}
|
||||
|
||||
return filtered
|
||||
}
|
||||
|
||||
// doTidy runs go mod tidy check.
|
||||
func doTidy() {
|
||||
targets := []string{"go.mod", "go.sum"}
|
||||
var tc = new(build.GoToolchain)
|
||||
|
||||
hashes, err := build.HashFiles(targets)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to hash go.mod/go.sum: %v", err)
|
||||
}
|
||||
build.MustRun(new(build.GoToolchain).Go("mod", "tidy"))
|
||||
|
||||
tidied, err := build.HashFiles(targets)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to rehash go.mod/go.sum: %v", err)
|
||||
}
|
||||
if updates := build.DiffHashes(hashes, tidied); len(updates) > 0 {
|
||||
log.Fatalf("files changed on running 'go mod tidy': %v", updates)
|
||||
for _, mod := range goModules {
|
||||
tidy := tc.Go("mod", "tidy", "-diff")
|
||||
tidy.Dir = mod
|
||||
build.MustRun(tidy)
|
||||
}
|
||||
fmt.Println("No untidy module files detected.")
|
||||
}
|
||||
|
|
@ -284,38 +310,79 @@ func doTidy() {
|
|||
func doGenerate() {
|
||||
var (
|
||||
cachedir = flag.String("cachedir", "./build/cache", "directory for caching binaries.")
|
||||
tc = new(build.GoToolchain)
|
||||
)
|
||||
// Compute the origin hashes of all the files
|
||||
var hashes map[string][32]byte
|
||||
|
||||
var err error
|
||||
hashes, err = build.HashFolder(".", []string{"tests/testdata", "build/cache"})
|
||||
if err != nil {
|
||||
log.Fatal("Error computing hashes", "err", err)
|
||||
}
|
||||
// Run any go generate steps we might be missing
|
||||
var (
|
||||
protocPath = downloadProtoc(*cachedir)
|
||||
protocGenGoPath = downloadProtocGenGo(*cachedir)
|
||||
)
|
||||
c := new(build.GoToolchain).Go("generate", "./...")
|
||||
pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")}
|
||||
c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
|
||||
build.MustRun(c)
|
||||
|
||||
// Check if generate file hashes have changed
|
||||
generated, err := build.HashFolder(".", []string{"tests/testdata", "build/cache"})
|
||||
if err != nil {
|
||||
log.Fatalf("Error re-computing hashes: %v", err)
|
||||
}
|
||||
updates := build.DiffHashes(hashes, generated)
|
||||
for _, file := range updates {
|
||||
log.Printf("File changed: %s", file)
|
||||
}
|
||||
if len(updates) != 0 {
|
||||
log.Fatal("One or more generated files were updated by running 'go generate ./...'")
|
||||
for _, mod := range goModules {
|
||||
// Compute the origin hashes of all the files
|
||||
hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"})
|
||||
if err != nil {
|
||||
log.Fatal("Error computing hashes", "err", err)
|
||||
}
|
||||
|
||||
c := tc.Go("generate", "./...")
|
||||
c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
|
||||
c.Dir = mod
|
||||
build.MustRun(c)
|
||||
// Check if generate file hashes have changed
|
||||
generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"})
|
||||
if err != nil {
|
||||
log.Fatalf("Error re-computing hashes: %v", err)
|
||||
}
|
||||
updates := build.DiffHashes(hashes, generated)
|
||||
for _, file := range updates {
|
||||
log.Printf("File changed: %s", file)
|
||||
}
|
||||
if len(updates) != 0 {
|
||||
log.Fatal("One or more generated files were updated by running 'go generate ./...'")
|
||||
}
|
||||
}
|
||||
fmt.Println("No stale files detected.")
|
||||
|
||||
// Run go mod tidy check.
|
||||
for _, mod := range goModules {
|
||||
tidy := tc.Go("mod", "tidy", "-diff")
|
||||
tidy.Dir = mod
|
||||
build.MustRun(tidy)
|
||||
}
|
||||
fmt.Println("No untidy module files detected.")
|
||||
}
|
||||
|
||||
// doBadDeps verifies whether certain unintended dependencies between some
|
||||
// packages leak into the codebase due to a refactor. This is not an exhaustive
|
||||
// list, rather something we build up over time at sensitive places.
|
||||
func doBadDeps() {
|
||||
baddeps := [][2]string{
|
||||
// Rawdb tends to be a dumping ground for db utils, sometimes leaking the db itself
|
||||
{"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"},
|
||||
{"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/pebbledb"},
|
||||
}
|
||||
tc := new(build.GoToolchain)
|
||||
|
||||
var failed bool
|
||||
for _, rule := range baddeps {
|
||||
out, err := tc.Go("list", "-deps", rule[0]).CombinedOutput()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to list '%s' dependencies: %v", rule[0], err)
|
||||
}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if strings.TrimSpace(line) == rule[1] {
|
||||
log.Printf("Found bad dependency '%s' -> '%s'", rule[0], rule[1])
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
log.Fatalf("Bad dependencies detected.")
|
||||
}
|
||||
fmt.Println("No bad dependencies detected.")
|
||||
}
|
||||
|
||||
// doLint runs golangci-lint on requested packages.
|
||||
|
|
@ -324,27 +391,42 @@ func doLint(cmdline []string) {
|
|||
cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.")
|
||||
)
|
||||
flag.CommandLine.Parse(cmdline)
|
||||
packages := []string{"./..."}
|
||||
if len(flag.CommandLine.Args()) > 0 {
|
||||
packages = flag.CommandLine.Args()
|
||||
}
|
||||
|
||||
linter := downloadLinter(*cachedir)
|
||||
lflags := []string{"run", "--config", ".golangci.yml"}
|
||||
build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
|
||||
linter, err := filepath.Abs(linter)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
config, err := filepath.Abs(".golangci.yml")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
lflags := []string{"run", "--config", config}
|
||||
packages := flag.CommandLine.Args()
|
||||
if len(packages) > 0 {
|
||||
build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
|
||||
} else {
|
||||
// Run for all modules in workspace.
|
||||
for _, mod := range goModules {
|
||||
args := append(lflags, "./...")
|
||||
lintcmd := exec.Command(linter, args...)
|
||||
lintcmd.Dir = mod
|
||||
build.MustRunWithOutput(lintcmd)
|
||||
}
|
||||
}
|
||||
fmt.Println("You have achieved perfection.")
|
||||
}
|
||||
|
||||
// downloadLinter downloads and unpacks golangci-lint.
|
||||
func downloadLinter(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "golangci")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("golangci")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
arch := runtime.GOARCH
|
||||
ext := ".tar.gz"
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
ext = ".zip"
|
||||
}
|
||||
|
|
@ -352,9 +434,8 @@ func downloadLinter(cachedir string) string {
|
|||
arch += "v" + os.Getenv("GOARM")
|
||||
}
|
||||
base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
|
||||
url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
|
||||
archivePath := filepath.Join(cachedir, base+ext)
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := build.ExtractArchive(archivePath, cachedir); err != nil {
|
||||
|
|
@ -363,40 +444,6 @@ func downloadLinter(cachedir string) string {
|
|||
return filepath.Join(cachedir, base, "golangci-lint")
|
||||
}
|
||||
|
||||
// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc
|
||||
// in the generate command. It returns the full path of the directory
|
||||
// containing the 'protoc-gen-go' executable.
|
||||
func downloadProtocGenGo(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "protoc-gen-go")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH)
|
||||
archiveName := baseName
|
||||
if runtime.GOOS == "windows" {
|
||||
archiveName += ".zip"
|
||||
} else {
|
||||
archiveName += ".tar.gz"
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName)
|
||||
|
||||
archivePath := filepath.Join(cachedir, archiveName)
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest := filepath.Join(cachedir, baseName)
|
||||
if err := build.ExtractArchive(archivePath, extractDest); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest, err = filepath.Abs(extractDest)
|
||||
if err != nil {
|
||||
log.Fatal("error resolving absolute path for protoc", "err", err)
|
||||
}
|
||||
return extractDest
|
||||
}
|
||||
|
||||
// protocArchiveBaseName returns the name of the protoc archive file for
|
||||
// the current system, stripped of version and file suffix.
|
||||
func protocArchiveBaseName() (string, error) {
|
||||
|
|
@ -420,12 +467,44 @@ func protocArchiveBaseName() (string, error) {
|
|||
}
|
||||
}
|
||||
|
||||
// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc
|
||||
// in the generate command. It returns the full path of the directory
|
||||
// containing the 'protoc-gen-go' executable.
|
||||
func downloadProtocGenGo(cachedir string) string {
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("protoc-gen-go")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH)
|
||||
archiveName := baseName
|
||||
if runtime.GOOS == "windows" {
|
||||
archiveName += ".zip"
|
||||
} else {
|
||||
archiveName += ".tar.gz"
|
||||
}
|
||||
|
||||
archivePath := path.Join(cachedir, archiveName)
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest := filepath.Join(cachedir, baseName)
|
||||
if err := build.ExtractArchive(archivePath, extractDest); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest, err = filepath.Abs(extractDest)
|
||||
if err != nil {
|
||||
log.Fatal("error resolving absolute path for protoc", "err", err)
|
||||
}
|
||||
return extractDest
|
||||
}
|
||||
|
||||
// downloadProtoc downloads the prebuilt protoc binary used to lint generated
|
||||
// files as a CI step. It returns the full path to the directory containing
|
||||
// the protoc executable.
|
||||
func downloadProtoc(cachedir string) string {
|
||||
csdb := build.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := build.Version(csdb, "protoc")
|
||||
csdb := download.MustLoadChecksums("build/checksums.txt")
|
||||
version, err := csdb.FindVersion("protoc")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -436,10 +515,8 @@ func downloadProtoc(cachedir string) string {
|
|||
|
||||
fileName := fmt.Sprintf("protoc-%s-%s", version, baseName)
|
||||
archiveFileName := fileName + ".zip"
|
||||
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName)
|
||||
archivePath := filepath.Join(cachedir, archiveFileName)
|
||||
|
||||
if err := csdb.DownloadFile(url, archivePath); err != nil {
|
||||
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
extractDest := filepath.Join(cachedir, fileName)
|
||||
|
|
@ -462,11 +539,12 @@ func doXgo(cmdline []string) {
|
|||
env := build.Env()
|
||||
|
||||
// Make sure xgo is available for cross compilation
|
||||
gogetxgo := goTool("get", "github.com/karalabe/xgo")
|
||||
tc := build.GoToolchain{}
|
||||
gogetxgo := tc.Go("get", "github.com/karalabe/xgo")
|
||||
build.MustRun(gogetxgo)
|
||||
|
||||
// If all tools building is requested, build everything the builder wants
|
||||
args := append(buildFlags(env), flag.Args()...)
|
||||
args := append(buildFlags(env, false, nil), flag.Args()...)
|
||||
|
||||
if *alltools {
|
||||
args = append(args, []string{"--dest", GOBIN}...)
|
||||
|
|
|
|||
|
|
@ -1,151 +0,0 @@
|
|||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package build
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ChecksumDB keeps file checksums.
|
||||
type ChecksumDB struct {
|
||||
allChecksums []string
|
||||
}
|
||||
|
||||
// MustLoadChecksums loads a file containing checksums.
|
||||
func MustLoadChecksums(file string) *ChecksumDB {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Fatal("can't load checksum file: " + err.Error())
|
||||
}
|
||||
return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")}
|
||||
}
|
||||
|
||||
// Verify checks whether the given file is valid according to the checksum database.
|
||||
func (db *ChecksumDB) Verify(path string) error {
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash := hex.EncodeToString(h.Sum(nil))
|
||||
if !db.findHash(filepath.Base(path), fileHash) {
|
||||
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *ChecksumDB) findHash(basename, hash string) bool {
|
||||
want := hash + " " + basename
|
||||
for _, line := range db.allChecksums {
|
||||
if strings.TrimSpace(line) == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DownloadFile downloads a file and verifies its checksum.
|
||||
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
|
||||
if err := db.Verify(dstPath); err == nil {
|
||||
fmt.Printf("%s is up-to-date\n", dstPath)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s is stale\n", dstPath)
|
||||
fmt.Printf("downloading from %s\n", url)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download error: status %d", resp.StatusCode)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := newDownloadWriter(fd, resp.ContentLength)
|
||||
_, err = io.Copy(dst, resp.Body)
|
||||
dst.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Verify(dstPath)
|
||||
}
|
||||
|
||||
type downloadWriter struct {
|
||||
file *os.File
|
||||
dstBuf *bufio.Writer
|
||||
size int64
|
||||
written int64
|
||||
lastpct int64
|
||||
}
|
||||
|
||||
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
|
||||
return &downloadWriter{
|
||||
file: dst,
|
||||
dstBuf: bufio.NewWriter(dst),
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Write(buf []byte) (int, error) {
|
||||
n, err := w.dstBuf.Write(buf)
|
||||
|
||||
// Report progress.
|
||||
w.written += int64(n)
|
||||
pct := w.written * 10 / w.size * 10
|
||||
if pct != w.lastpct {
|
||||
if w.lastpct != 0 {
|
||||
fmt.Print("...")
|
||||
}
|
||||
fmt.Print(pct, "%")
|
||||
w.lastpct = pct
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Close() error {
|
||||
if w.lastpct > 0 {
|
||||
fmt.Println() // Finish the progress line.
|
||||
}
|
||||
flushErr := w.dstBuf.Flush()
|
||||
closeErr := w.file.Close()
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
|
|
@ -25,32 +25,6 @@ import (
|
|||
"strings"
|
||||
)
|
||||
|
||||
// FileExist checks if a file exists at path.
|
||||
func FileExist(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// HashFiles iterates the provided set of files, computing the hash of each.
|
||||
func HashFiles(files []string) (map[string][32]byte, error) {
|
||||
res := make(map[string][32]byte)
|
||||
for _, filePath := range files {
|
||||
f, err := os.OpenFile(filePath, os.O_RDONLY, 0666)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hasher := sha256.New()
|
||||
if _, err := io.Copy(hasher, f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res[filePath] = [32]byte(hasher.Sum(nil))
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// HashFolder iterates all files under the given directory, computing the hash
|
||||
// of each.
|
||||
func HashFolder(folder string, exlude []string) (map[string][32]byte, error) {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import (
|
|||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/XinFinOrg/XDPoSChain/internal/download"
|
||||
)
|
||||
|
||||
type GoToolchain struct {
|
||||
|
|
@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd {
|
|||
|
||||
// DownloadGo downloads the Go binary distribution and unpacks it into a temporary
|
||||
// directory. It returns the GOROOT of the unpacked toolchain.
|
||||
func DownloadGo(csdb *ChecksumDB) string {
|
||||
version, err := Version(csdb, "golang")
|
||||
func DownloadGo(csdb *download.ChecksumDB) string {
|
||||
version, err := csdb.FindVersion("golang")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
|
@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string {
|
|||
}
|
||||
return goroot
|
||||
}
|
||||
|
||||
// Version returns the versions defined in the checksumdb.
|
||||
func Version(csdb *ChecksumDB, version string) (string, error) {
|
||||
for _, l := range csdb.allChecksums {
|
||||
if !strings.HasPrefix(l, "# version:") {
|
||||
continue
|
||||
}
|
||||
v := strings.Split(l, ":")[1]
|
||||
parts := strings.Split(v, " ")
|
||||
if len(parts) != 2 {
|
||||
log.Print("Erroneous version-string", "v", l)
|
||||
continue
|
||||
}
|
||||
if parts[0] == version {
|
||||
return parts[1], nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no version found for '%v'", version)
|
||||
}
|
||||
|
||||
// DownloadAndVerifyChecksums downloads all files and checks that they match
|
||||
// the checksum given in checksums.txt.
|
||||
// This task can be used to sanity-check new checksums.
|
||||
func DownloadAndVerifyChecksums(csdb *ChecksumDB) {
|
||||
var (
|
||||
base = ""
|
||||
ucache = os.TempDir()
|
||||
)
|
||||
for _, l := range csdb.allChecksums {
|
||||
if strings.HasPrefix(l, "# https://") {
|
||||
base = l[2:]
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(l, "#") {
|
||||
continue
|
||||
}
|
||||
hashFile := strings.Split(l, " ")
|
||||
if len(hashFile) != 2 {
|
||||
continue
|
||||
}
|
||||
file := hashFile[1]
|
||||
url := base + file
|
||||
dst := filepath.Join(ucache, file)
|
||||
if err := csdb.DownloadFile(url, dst); err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,16 +17,18 @@
|
|||
package build
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -35,6 +37,9 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
|
|||
// MustRun executes the given command and exits the host process for
|
||||
// any error.
|
||||
func MustRun(cmd *exec.Cmd) {
|
||||
if cmd.Dir != "" && cmd.Dir != "." {
|
||||
fmt.Printf("(in %s) ", cmd.Dir)
|
||||
}
|
||||
fmt.Println(">>>", printArgs(cmd.Args))
|
||||
if !*DryRunFlag {
|
||||
cmd.Stderr = os.Stderr
|
||||
|
|
@ -67,6 +72,13 @@ func MustRunCommand(cmd string, args ...string) {
|
|||
// printed while it runs. This is useful for CI builds where the process will be stopped
|
||||
// when there is no output.
|
||||
func MustRunCommandWithOutput(cmd string, args ...string) {
|
||||
MustRunWithOutput(exec.Command(cmd, args...))
|
||||
}
|
||||
|
||||
// MustRunWithOutput runs the given command, and ensures that some output will be printed
|
||||
// while it runs. This is useful for CI builds where the process will be stopped when
|
||||
// there is no output.
|
||||
func MustRunWithOutput(cmd *exec.Cmd) {
|
||||
interval := time.NewTicker(time.Minute)
|
||||
done := make(chan struct{})
|
||||
defer interval.Stop()
|
||||
|
|
@ -81,16 +93,7 @@ func MustRunCommandWithOutput(cmd string, args ...string) {
|
|||
}
|
||||
}
|
||||
}()
|
||||
MustRun(exec.Command(cmd, args...))
|
||||
}
|
||||
|
||||
// GOPATH returns the value that the GOPATH environment
|
||||
// variable should be set to.
|
||||
func GOPATH() string {
|
||||
if os.Getenv("GOPATH") == "" {
|
||||
log.Fatal("GOPATH is not set")
|
||||
}
|
||||
return os.Getenv("GOPATH")
|
||||
MustRun(cmd)
|
||||
}
|
||||
|
||||
var warnedAboutGit bool
|
||||
|
|
@ -101,13 +104,14 @@ func RunGit(args ...string) string {
|
|||
cmd := exec.Command("git", args...)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
if err := cmd.Run(); err == exec.ErrNotFound {
|
||||
if !warnedAboutGit {
|
||||
log.Println("Warning: can't find 'git' in PATH")
|
||||
warnedAboutGit = true
|
||||
if err := cmd.Run(); err != nil {
|
||||
if e, ok := err.(*exec.Error); ok && e.Err == exec.ErrNotFound {
|
||||
if !warnedAboutGit {
|
||||
log.Println("Warning: can't find 'git' in PATH")
|
||||
warnedAboutGit = true
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
} else if err != nil {
|
||||
log.Fatal(strings.Join(cmd.Args, " "), ": ", err, "\n", stderr.String())
|
||||
}
|
||||
return strings.TrimSpace(stdout.String())
|
||||
|
|
@ -122,46 +126,136 @@ func readGitFile(file string) string {
|
|||
return strings.TrimSpace(string(content))
|
||||
}
|
||||
|
||||
// GoTool returns the command that runs a go tool. This uses go from GOROOT instead of PATH
|
||||
// so that go commands executed by build use the same version of Go as the 'host' that runs
|
||||
// build code. e.g.
|
||||
//
|
||||
// /usr/lib/go-1.11/bin/go run build/ci.go ...
|
||||
//
|
||||
// runs using go 1.11 and invokes go 1.11 tools from the same GOROOT. This is also important
|
||||
// because runtime.Version checks on the host should match the tools that are run.
|
||||
func GoTool(tool string, args ...string) *exec.Cmd {
|
||||
args = append([]string{tool}, args...)
|
||||
return exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...)
|
||||
// Render renders the given template file into outputFile.
|
||||
func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) {
|
||||
tpl := template.Must(template.ParseFiles(templateFile))
|
||||
render(tpl, outputFile, outputPerm, x)
|
||||
}
|
||||
|
||||
// ExpandPackages expands a cmd/go import path pattern, skip vendor
|
||||
// packages, and skip time-consuming tests according to quick flag.
|
||||
func ExpandPackages(patterns []string, quick bool) []string {
|
||||
expand := false
|
||||
for _, pkg := range patterns {
|
||||
if strings.Contains(pkg, "...") {
|
||||
expand = true
|
||||
}
|
||||
}
|
||||
if expand {
|
||||
cmd := GoTool("list", patterns...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
log.Fatalf("package listing failed: %v\n%s", err, string(out))
|
||||
}
|
||||
var packages []string
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
if strings.Contains(line, "/vendor/") {
|
||||
continue
|
||||
}
|
||||
if quick && strings.Contains(line, "/consensus/tests/engine_v2_tests") {
|
||||
continue
|
||||
}
|
||||
packages = append(packages, strings.TrimSpace(line))
|
||||
|
||||
}
|
||||
return packages
|
||||
}
|
||||
return patterns
|
||||
// RenderString renders the given template string into outputFile.
|
||||
func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) {
|
||||
tpl := template.Must(template.New("").Parse(templateContent))
|
||||
render(tpl, outputFile, outputPerm, x)
|
||||
}
|
||||
|
||||
func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x interface{}) {
|
||||
if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
out, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, outputPerm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := tpl.Execute(out, x); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// UploadSFTP uploads files to a remote host using the sftp command line tool.
|
||||
// The destination host may be specified either as [user@]host: or as a URI in
|
||||
// the form sftp://[user@]host[:port].
|
||||
func UploadSFTP(identityFile, host, dir string, files []string) error {
|
||||
sftp := exec.Command("sftp")
|
||||
sftp.Stderr = os.Stderr
|
||||
if identityFile != "" {
|
||||
sftp.Args = append(sftp.Args, "-i", identityFile)
|
||||
}
|
||||
sftp.Args = append(sftp.Args, host)
|
||||
fmt.Println(">>>", printArgs(sftp.Args))
|
||||
if *DryRunFlag {
|
||||
return nil
|
||||
}
|
||||
|
||||
stdin, err := sftp.StdinPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't create stdin pipe for sftp: %v", err)
|
||||
}
|
||||
stdout, err := sftp.StdoutPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't create stdout pipe for sftp: %v", err)
|
||||
}
|
||||
if err := sftp.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
in := io.MultiWriter(stdin, os.Stdout)
|
||||
for _, f := range files {
|
||||
fmt.Fprintln(in, "put", f, filepath.Join(dir, filepath.Base(f)))
|
||||
}
|
||||
fmt.Fprintln(in, "exit")
|
||||
// Some issue with the PPA sftp server makes it so the server does not
|
||||
// respond properly to a 'bye', 'exit' or 'quit' from the client.
|
||||
// To work around that, we check the output, and when we see the client
|
||||
// exit command, we do a hard exit.
|
||||
// See
|
||||
// https://github.com/kolban-google/sftp-gcs/issues/23
|
||||
// https://github.com/mscdex/ssh2/pull/1111
|
||||
aborted := false
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
txt := scanner.Text()
|
||||
fmt.Println(txt)
|
||||
if txt == "sftp> exit" {
|
||||
// Give it .5 seconds to exit (server might be fixed), then
|
||||
// hard kill it from the outside
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
aborted = true
|
||||
sftp.Process.Kill()
|
||||
}
|
||||
}
|
||||
}()
|
||||
stdin.Close()
|
||||
err = sftp.Wait()
|
||||
if aborted {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// FindMainPackages finds all 'main' packages in the given directory and returns their
|
||||
// package paths.
|
||||
func FindMainPackages(tc *GoToolchain, pattern string) []string {
|
||||
list := tc.Go("list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, pattern)
|
||||
output, err := list.Output()
|
||||
if err != nil {
|
||||
log.Fatal("go list failed:", err)
|
||||
}
|
||||
var result []string
|
||||
for l := range bytes.Lines(output) {
|
||||
l = bytes.TrimSpace(l)
|
||||
if len(l) > 0 {
|
||||
result = append(result, string(l))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// FindAllPackages expands a cmd/go import path pattern.
|
||||
func FindAllPackages(tc *GoToolchain) []string {
|
||||
list := tc.Go("list", "./...")
|
||||
output, err := list.Output()
|
||||
if err != nil {
|
||||
log.Fatal("go list failed:", err)
|
||||
}
|
||||
result := make([]string, 0, len(output))
|
||||
for line := range bytes.Lines(output) {
|
||||
pkg := bytes.TrimSpace(line)
|
||||
if len(pkg) > 0 {
|
||||
result = append(result, string(pkg))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GOPATH returns the value that the GOPATH environment
|
||||
// variable should be set to.
|
||||
func GOPATH() string {
|
||||
gopath := os.Getenv("GOPATH")
|
||||
if gopath == "" {
|
||||
log.Fatal("GOPATH is not set")
|
||||
}
|
||||
return gopath
|
||||
}
|
||||
|
|
|
|||
298
internal/download/download.go
Normal file
298
internal/download/download.go
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Package download implements checksum-verified file downloads.
|
||||
package download
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"iter"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ChecksumDB keeps file checksums and tool versions.
|
||||
type ChecksumDB struct {
|
||||
hashes []hashEntry
|
||||
versions []versionEntry
|
||||
}
|
||||
|
||||
type versionEntry struct {
|
||||
name string
|
||||
version string
|
||||
}
|
||||
|
||||
type hashEntry struct {
|
||||
hash string
|
||||
file string
|
||||
url *url.URL
|
||||
}
|
||||
|
||||
// MustLoadChecksums loads a file containing checksums.
|
||||
func MustLoadChecksums(file string) *ChecksumDB {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
panic("can't load checksum file: " + err.Error())
|
||||
}
|
||||
db, err := ParseChecksums(content)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("invalid checksums in %s: %v", file, err))
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// ParseChecksums parses a checksum database.
|
||||
func ParseChecksums(input []byte) (*ChecksumDB, error) {
|
||||
var (
|
||||
csdb = new(ChecksumDB)
|
||||
rd = bytes.NewBuffer(input)
|
||||
lastURL *url.URL
|
||||
)
|
||||
for lineNum := 1; ; lineNum++ {
|
||||
line, err := rd.ReadString('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case line == "":
|
||||
// Blank lines are allowed, and they reset the current urlEntry.
|
||||
lastURL = nil
|
||||
|
||||
case strings.HasPrefix(line, "#"):
|
||||
// It's a comment. Some comments have special meaning.
|
||||
content := strings.TrimLeft(line, "# ")
|
||||
switch {
|
||||
case strings.HasPrefix(content, "version:"):
|
||||
// Version comments define the version of a tool.
|
||||
v := strings.Split(content, ":")[1]
|
||||
parts := strings.Split(v, " ")
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v)
|
||||
}
|
||||
csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]})
|
||||
|
||||
case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"):
|
||||
// URL comments define the URL where the following files are found. Here
|
||||
// we keep track of the last found urlEntry and attach it to each file later.
|
||||
u, err := url.Parse(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: invalid URL: %v", lineNum, err)
|
||||
}
|
||||
lastURL = u
|
||||
}
|
||||
|
||||
default:
|
||||
// It's a file hash entry.
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields))
|
||||
}
|
||||
csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL})
|
||||
}
|
||||
}
|
||||
return csdb, nil
|
||||
}
|
||||
|
||||
// Files returns an iterator over all file names.
|
||||
func (db *ChecksumDB) Files() iter.Seq[string] {
|
||||
return func(yield func(string) bool) {
|
||||
for _, e := range db.hashes {
|
||||
if !yield(e.file) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DownloadAndVerifyAll downloads all files and checks that they match the checksum given in
|
||||
// the database. This task can be used to sanity-check new checksums.
|
||||
func (db *ChecksumDB) DownloadAndVerifyAll() {
|
||||
var tmp = os.TempDir()
|
||||
for _, e := range db.hashes {
|
||||
if e.url == nil {
|
||||
fmt.Printf("Skipping verification of %s: no URL defined in checksum database", e.file)
|
||||
continue
|
||||
}
|
||||
url := e.url.JoinPath(e.file).String()
|
||||
dst := filepath.Join(tmp, e.file)
|
||||
if err := db.DownloadFile(url, dst); err != nil {
|
||||
fmt.Println("error:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verifyHash checks that the file at 'path' has the expected hash.
|
||||
func verifyHash(path, expectedHash string) error {
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
|
||||
return err
|
||||
}
|
||||
fileHash := hex.EncodeToString(h.Sum(nil))
|
||||
if fileHash != expectedHash {
|
||||
return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFileFromKnownURL downloads a file from the URL defined in the checksum database.
|
||||
func (db *ChecksumDB) DownloadFileFromKnownURL(dstPath string) error {
|
||||
base := filepath.Base(dstPath)
|
||||
url, err := db.FindURL(base)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.DownloadFile(url, dstPath)
|
||||
}
|
||||
|
||||
// DownloadFile downloads a file and verifies its checksum.
|
||||
func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
|
||||
basename := filepath.Base(dstPath)
|
||||
hash := db.findHash(basename)
|
||||
if hash == "" {
|
||||
return fmt.Errorf("no known hash for file %q", basename)
|
||||
}
|
||||
// Shortcut if already downloaded.
|
||||
if verifyHash(dstPath, hash) == nil {
|
||||
fmt.Printf("%s is up-to-date\n", dstPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%s is stale\n", dstPath)
|
||||
fmt.Printf("downloading from %s\n", url)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download error: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download error: status %d", resp.StatusCode)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Download to a temporary file.
|
||||
tmpfile := dstPath + ".tmp"
|
||||
fd, err := os.OpenFile(tmpfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := newDownloadWriter(fd, resp.ContentLength)
|
||||
_, err = io.Copy(dst, resp.Body)
|
||||
dst.Close()
|
||||
if err != nil {
|
||||
os.Remove(tmpfile)
|
||||
return err
|
||||
}
|
||||
if err := verifyHash(tmpfile, hash); err != nil {
|
||||
os.Remove(tmpfile)
|
||||
return err
|
||||
}
|
||||
// It's valid, rename to dstPath to complete the download.
|
||||
return os.Rename(tmpfile, dstPath)
|
||||
}
|
||||
|
||||
// findHash returns the known hash of a file.
|
||||
func (db *ChecksumDB) findHash(basename string) string {
|
||||
for _, e := range db.hashes {
|
||||
if e.file == basename {
|
||||
return e.hash
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FindVersion returns the current known version of a tool, if it is defined in the file.
|
||||
func (db *ChecksumDB) FindVersion(tool string) (string, error) {
|
||||
for _, e := range db.versions {
|
||||
if e.name == tool {
|
||||
return e.version, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("tool version %q not defined in checksum database", tool)
|
||||
}
|
||||
|
||||
// FindURL gets the URL for a file.
|
||||
func (db *ChecksumDB) FindURL(basename string) (string, error) {
|
||||
for _, e := range db.hashes {
|
||||
if e.file == basename {
|
||||
if e.url == nil {
|
||||
return "", fmt.Errorf("file %q has no URL defined", e.file)
|
||||
}
|
||||
return e.url.JoinPath(e.file).String(), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("file %q does not exist in checksum database", basename)
|
||||
}
|
||||
|
||||
type downloadWriter struct {
|
||||
file *os.File
|
||||
dstBuf *bufio.Writer
|
||||
size int64
|
||||
written int64
|
||||
lastpct int64
|
||||
}
|
||||
|
||||
func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
|
||||
return &downloadWriter{
|
||||
file: dst,
|
||||
dstBuf: bufio.NewWriter(dst),
|
||||
size: size,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Write(buf []byte) (int, error) {
|
||||
n, err := w.dstBuf.Write(buf)
|
||||
|
||||
// Report progress.
|
||||
w.written += int64(n)
|
||||
pct := w.written * 10 / w.size * 10
|
||||
if pct != w.lastpct {
|
||||
if w.lastpct != 0 {
|
||||
fmt.Print("...")
|
||||
}
|
||||
fmt.Print(pct, "%")
|
||||
w.lastpct = pct
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *downloadWriter) Close() error {
|
||||
if w.lastpct > 0 {
|
||||
fmt.Println() // Finish the progress line.
|
||||
}
|
||||
flushErr := w.dstBuf.Flush()
|
||||
closeErr := w.file.Close()
|
||||
if flushErr != nil {
|
||||
return flushErr
|
||||
}
|
||||
return closeErr
|
||||
}
|
||||
Loading…
Reference in a new issue