diff --git a/Makefile b/Makefile index 8a0faf49f7..2800b9605f 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ all: test: all go run build/ci.go test -failfast -#? quick-test: Run the tests except time-consuming tests. +#? quick-test: Run the tests except time-consuming packages. quick-test: all go run build/ci.go test --quick -failfast @@ -53,14 +53,18 @@ quick-test: all lint: ## Run linters. $(GORUN) build/ci.go lint -#? tidy: Verify go.mod and go.sum by 'go mod tidy' +#? tidy: Verify go.mod and go.sum are updated. tidy: ## Run 'go mod tidy'. $(GORUN) build/ci.go tidy -#? generate: Verify everything is 'go generate'-ed +#? generate: Verify everything is 'go generate'. generate: ## Run 'go generate ./...'. $(GORUN) build/ci.go generate +#? baddeps: Verify certain dependencies are avoided. +baddeps: + $(GORUN) build/ci.go baddeps + #? fmt: Ensure consistent code formatting. fmt: gofmt -s -w $(shell find . -name "*.go") diff --git a/build/ci.go b/build/ci.go index ae82bd1922..a977088668 100644 --- a/build/ci.go +++ b/build/ci.go @@ -24,14 +24,15 @@ Usage: go run build/ci.go Available commands are: - lint -- runs certain pre-selected linters - tidy -- verifies that everything is 'go mod tidy'-ed - generate -- verifies that everything is 'go generate'-ed + lint -- runs certain pre-selected linters + tidy -- verifies that everything is 'go mod tidy'-ed + generate -- verifies that everything is 'go generate'-ed + baddeps -- verifies that certain dependencies are avoided - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - importkeys -- imports signing keys from env - xgo [ -alltools ] [ options ] -- cross builds according to options + install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + importkeys -- imports signing keys from env + xgo [ -alltools ] [ options ] -- cross builds according to options For all commands, -n prevents execution of external programs (dry run mode). */ @@ -40,31 +41,36 @@ package main import ( "flag" "fmt" - "go/parser" - "go/token" "log" "os" "os/exec" + "path" "path/filepath" "runtime" + "slices" "strings" "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/internal/build" + "github.com/XinFinOrg/XDPoSChain/internal/download" ) var ( + goModules = []string{ + ".", + } + // Files that end up in the geth-alltools*.zip archive. allToolsArchiveFiles = []string{ "COPYING", executablePath("abigen"), executablePath("bootnode"), + executablePath("ethkey"), executablePath("evm"), - executablePath("geth"), + executablePath("p2psim"), executablePath("puppeth"), executablePath("rlpdump"), - executablePath("swarm"), - executablePath("wnode"), + executablePath("XDC"), } ) @@ -97,6 +103,8 @@ func main() { doTidy() case "generate": doGenerate() + case "baddeps": + doBadDeps() case "xgo": doXgo(os.Args[2:]) default: @@ -108,145 +116,131 @@ func main() { func doInstall(cmdline []string) { var ( - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) env := build.Env() - // Check Go version. People regularly open issues about compilation - // failure with outdated Go. This should save them the trouble. - if !strings.Contains(runtime.Version(), "devel") { - // Figure out the minor version number since we can't textually compare (1.10 < 1.9) - var minor int - fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) + // Configure the toolchain. + tc := build.GoToolchain{GOARCH: *arch, CC: *cc} + if *dlgo { + csdb := download.MustLoadChecksums("build/checksums.txt") + tc.Root = build.DownloadGo(csdb) + } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} - if minor < 25 { - log.Println("You have Go version", runtime.Version()) - log.Println("XDC requires at least Go version 1.25 and cannot") - log.Println("be compiled with an earlier version. Please upgrade your Go installation.") - os.Exit(1) - } - } - // Compile packages given as arguments, or everything if there are no arguments. - packages := []string{"./..."} - if flag.NArg() > 0 { - packages = flag.Args() - } - // packages = build.ExpandPackagesNoVendor(packages) + // Configure the build. + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) - if *arch == "" || *arch == runtime.GOARCH { - goinstall := goTool("install", buildFlags(env)...) - goinstall.Args = append(goinstall.Args, "-v") - goinstall.Args = append(goinstall.Args, packages...) - build.MustRun(goinstall) - return - } - // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds - if *arch == "arm" { - os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) - for _, path := range filepath.SplitList(build.GOPATH()) { - os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) - } - } - // Seems we are cross compiling, work around forbidden GOBIN - goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) - goinstall.Args = append(goinstall.Args, "-v") - goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...) - goinstall.Args = append(goinstall.Args, packages...) - build.MustRun(goinstall) + // Show packages during build. + gobuild.Args = append(gobuild.Args, "-v") - if cmds, err := os.ReadDir("cmd"); err == nil { - for _, cmd := range cmds { - pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly) - if err != nil { - log.Fatal(err) - } - for name := range pkgs { - if name == "main" { - gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...) - gobuild.Args = append(gobuild.Args, "-v") - gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...) - gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name())) - build.MustRun(gobuild) - break - } - } - } + // Now we choose what we're even building. + // Default: collect all 'main' packages in cmd/ and build those. + packages := flag.Args() + if len(packages) == 0 { + // NOTE: to collect all main packages, use: + // packages = build.FindMainPackages(&tc, "./...") + packages = build.FindMainPackages(&tc, "./cmd/...") + } + + // Do the build! + for _, pkg := range packages { + args := slices.Clone(gobuild.Args) + args = append(args, "-o", executablePath(path.Base(pkg))) + args = append(args, pkg) + build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env}) } } -func buildFlags(env build.Environment) (flags []string) { +// buildFlags returns the go tool flags for building. +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string + // See https://github.com/golang/go/issues/33772#issuecomment-528176001 + // We need to set --buildid to the linker here, and also pass --build-id to the + // cgo-linker further down. + ld = append(ld, "--buildid=none") if env.Commit != "" { ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitCommit="+env.Commit) ld = append(ld, "-X", "github.com/XinFinOrg/XDPoSChain/internal/version.gitDate="+env.Date) } + // Strip DWARF on darwin. This used to be required for certain things, + // and there is no downside to this, so we just keep doing it. if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - + if runtime.GOOS == "linux" { + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + // See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options + // regarding the options --build-id=none and --strip-all. It is needed for + // reproducible builds; removing references to temporary files in C-land, and + // making build-id reproducibly absent. + extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") + } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } + // We use -trimpath to avoid leaking local paths into the built executables. + flags = append(flags, "-trimpath") return flags } -func goTool(subcmd string, args ...string) *exec.Cmd { - return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...) -} - -func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { - cmd := build.GoTool(subcmd, args...) - cmd.Env = []string{"GOPATH=" + build.GOPATH()} - if arch == "" || arch == runtime.GOARCH { - cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) - } else { - cmd.Env = append(cmd.Env, "CGO_ENABLED=1") - cmd.Env = append(cmd.Env, "GOARCH="+arch) - } - if cc != "" { - cmd.Env = append(cmd.Env, "CC="+cc) - } - for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { - continue - } - cmd.Env = append(cmd.Env, e) - } - return cmd -} - // Running The Tests // // "tests" also includes static analysis tools such as vet. -func doTest(cmdline []string) { - coverage := flag.Bool("coverage", false, "Whether to record code coverage") - verbose := flag.Bool("v", false, "Whether to log verbosely") - quick := flag.Bool("quick", false, "Whether to skip long time test") - failfast := flag.Bool("failfast", false, "Do not start new tests after the first test failure") - flag.CommandLine.Parse(cmdline) - env := build.Env() - packages := []string{"./..."} // if a package has no test files, the lines in that package are not added to the total line count - if len(flag.CommandLine.Args()) > 0 { - packages = flag.CommandLine.Args() - } else { - // added all files in all packages (except vendor) to coverage report files count, even there is no test file in the package - packages = build.ExpandPackages(packages, *quick) +func doTest(cmdline []string) { + var ( + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Run tests for given architecture") + cc = flag.String("cc", "", "Sets C compiler binary") + coverage = flag.Bool("coverage", false, "Whether to record code coverage") + verbose = flag.Bool("v", false, "Whether to log verbosely") + race = flag.Bool("race", false, "Execute the race detector") + short = flag.Bool("short", false, "Pass the 'short'-flag to go test") + threads = flag.Int("p", 1, "Number of CPU threads to use for testing") + quick = flag.Bool("quick", false, "Whether to skip long time test") + failfast = flag.Bool("failfast", false, "Do not start new tests after the first test failure") + ) + flag.CommandLine.Parse(cmdline) + + // Load checksums file (needed for both spec tests and dlgo) + csdb := download.MustLoadChecksums("build/checksums.txt") + + // Configure the toolchain. + tc := build.GoToolchain{GOARCH: *arch, CC: *cc} + if *dlgo { + tc.Root = build.DownloadGo(csdb) } - // Run analysis tools before the tests. - // build.MustRun(goTool("vet", packages...)) + gotest := tc.Go("test") + + // CI needs a bit more time for the statetests (default 45m). + gotest.Args = append(gotest.Args, "-timeout=45m") + + // Enable integration-tests + gotest.Args = append(gotest.Args, "-tags=integrationtests") - // Run the actual tests. - // gotest := goTool("test", buildFlags(env)...) // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. - gotest := goTool("test", buildFlags(env)...) - gotest.Args = append(gotest.Args, "-p", "1") + gotest.Args = append(gotest.Args, "-p", fmt.Sprintf("%d", *threads)) if *coverage { - gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover", "-coverprofile=coverage.txt") + gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") } if *verbose { gotest.Args = append(gotest.Args, "-v") @@ -254,27 +248,59 @@ func doTest(cmdline []string) { if *failfast { gotest.Args = append(gotest.Args, "-failfast") } + if *race { + gotest.Args = append(gotest.Args, "-race") + } + if *short { + gotest.Args = append(gotest.Args, "-short") + } + packages := flag.CommandLine.Args() + if len(packages) > 0 { + if *quick { + packages = filterPackages(packages) + } + gotest.Args = append(gotest.Args, packages...) + build.MustRun(gotest) + return + } + + // No packages specified, run all tests for all modules. + if *quick { + packages = filterPackages(build.FindAllPackages(&tc)) + } else { + packages = []string{"./..."} + } gotest.Args = append(gotest.Args, packages...) - build.MustRun(gotest) + for _, mod := range goModules { + test := *gotest + test.Dir = mod + build.MustRun(&test) + } } -// doTidy assets that the Go modules files are tidied already. +// filterPackages removes time-consuming packages. +func filterPackages(packages []string) []string { + var filtered []string + + for _, pkg := range packages { + if strings.Contains(pkg, "/consensus/tests/engine_v2_tests") { + continue + } + filtered = append(filtered, pkg) + } + + return filtered +} + +// doTidy runs go mod tidy check. func doTidy() { - targets := []string{"go.mod", "go.sum"} + var tc = new(build.GoToolchain) - hashes, err := build.HashFiles(targets) - if err != nil { - log.Fatalf("failed to hash go.mod/go.sum: %v", err) - } - build.MustRun(new(build.GoToolchain).Go("mod", "tidy")) - - tidied, err := build.HashFiles(targets) - if err != nil { - log.Fatalf("failed to rehash go.mod/go.sum: %v", err) - } - if updates := build.DiffHashes(hashes, tidied); len(updates) > 0 { - log.Fatalf("files changed on running 'go mod tidy': %v", updates) + for _, mod := range goModules { + tidy := tc.Go("mod", "tidy", "-diff") + tidy.Dir = mod + build.MustRun(tidy) } fmt.Println("No untidy module files detected.") } @@ -284,38 +310,79 @@ func doTidy() { func doGenerate() { var ( cachedir = flag.String("cachedir", "./build/cache", "directory for caching binaries.") + tc = new(build.GoToolchain) ) - // Compute the origin hashes of all the files - var hashes map[string][32]byte - var err error - hashes, err = build.HashFolder(".", []string{"tests/testdata", "build/cache"}) - if err != nil { - log.Fatal("Error computing hashes", "err", err) - } // Run any go generate steps we might be missing var ( protocPath = downloadProtoc(*cachedir) protocGenGoPath = downloadProtocGenGo(*cachedir) ) - c := new(build.GoToolchain).Go("generate", "./...") pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} - c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator))) - build.MustRun(c) - // Check if generate file hashes have changed - generated, err := build.HashFolder(".", []string{"tests/testdata", "build/cache"}) - if err != nil { - log.Fatalf("Error re-computing hashes: %v", err) - } - updates := build.DiffHashes(hashes, generated) - for _, file := range updates { - log.Printf("File changed: %s", file) - } - if len(updates) != 0 { - log.Fatal("One or more generated files were updated by running 'go generate ./...'") + for _, mod := range goModules { + // Compute the origin hashes of all the files + hashes, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + if err != nil { + log.Fatal("Error computing hashes", "err", err) + } + + c := tc.Go("generate", "./...") + c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator))) + c.Dir = mod + build.MustRun(c) + // Check if generate file hashes have changed + generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"}) + if err != nil { + log.Fatalf("Error re-computing hashes: %v", err) + } + updates := build.DiffHashes(hashes, generated) + for _, file := range updates { + log.Printf("File changed: %s", file) + } + if len(updates) != 0 { + log.Fatal("One or more generated files were updated by running 'go generate ./...'") + } } fmt.Println("No stale files detected.") + + // Run go mod tidy check. + for _, mod := range goModules { + tidy := tc.Go("mod", "tidy", "-diff") + tidy.Dir = mod + build.MustRun(tidy) + } + fmt.Println("No untidy module files detected.") +} + +// doBadDeps verifies whether certain unintended dependencies between some +// packages leak into the codebase due to a refactor. This is not an exhaustive +// list, rather something we build up over time at sensitive places. +func doBadDeps() { + baddeps := [][2]string{ + // Rawdb tends to be a dumping ground for db utils, sometimes leaking the db itself + {"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"}, + {"github.com/XinFinOrg/XDPoSChain/core/rawdb", "github.com/XinFinOrg/XDPoSChain/ethdb/pebbledb"}, + } + tc := new(build.GoToolchain) + + var failed bool + for _, rule := range baddeps { + out, err := tc.Go("list", "-deps", rule[0]).CombinedOutput() + if err != nil { + log.Fatalf("Failed to list '%s' dependencies: %v", rule[0], err) + } + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == rule[1] { + log.Printf("Found bad dependency '%s' -> '%s'", rule[0], rule[1]) + failed = true + } + } + } + if failed { + log.Fatalf("Bad dependencies detected.") + } + fmt.Println("No bad dependencies detected.") } // doLint runs golangci-lint on requested packages. @@ -324,27 +391,42 @@ func doLint(cmdline []string) { cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") ) flag.CommandLine.Parse(cmdline) - packages := []string{"./..."} - if len(flag.CommandLine.Args()) > 0 { - packages = flag.CommandLine.Args() - } linter := downloadLinter(*cachedir) - lflags := []string{"run", "--config", ".golangci.yml"} - build.MustRunCommandWithOutput(linter, append(lflags, packages...)...) + linter, err := filepath.Abs(linter) + if err != nil { + log.Fatal(err) + } + config, err := filepath.Abs(".golangci.yml") + if err != nil { + log.Fatal(err) + } + + lflags := []string{"run", "--config", config} + packages := flag.CommandLine.Args() + if len(packages) > 0 { + build.MustRunCommandWithOutput(linter, append(lflags, packages...)...) + } else { + // Run for all modules in workspace. + for _, mod := range goModules { + args := append(lflags, "./...") + lintcmd := exec.Command(linter, args...) + lintcmd.Dir = mod + build.MustRunWithOutput(lintcmd) + } + } fmt.Println("You have achieved perfection.") } // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "golangci") + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("golangci") if err != nil { log.Fatal(err) } arch := runtime.GOARCH ext := ".tar.gz" - if runtime.GOOS == "windows" { ext = ".zip" } @@ -352,9 +434,8 @@ func downloadLinter(cachedir string) string { arch += "v" + os.Getenv("GOARM") } base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) - url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext) archivePath := filepath.Join(cachedir, base+ext) - if err := csdb.DownloadFile(url, archivePath); err != nil { + if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil { log.Fatal(err) } if err := build.ExtractArchive(archivePath, cachedir); err != nil { @@ -363,40 +444,6 @@ func downloadLinter(cachedir string) string { return filepath.Join(cachedir, base, "golangci-lint") } -// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc -// in the generate command. It returns the full path of the directory -// containing the 'protoc-gen-go' executable. -func downloadProtocGenGo(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "protoc-gen-go") - if err != nil { - log.Fatal(err) - } - baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH) - archiveName := baseName - if runtime.GOOS == "windows" { - archiveName += ".zip" - } else { - archiveName += ".tar.gz" - } - - url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName) - - archivePath := filepath.Join(cachedir, archiveName) - if err := csdb.DownloadFile(url, archivePath); err != nil { - log.Fatal(err) - } - extractDest := filepath.Join(cachedir, baseName) - if err := build.ExtractArchive(archivePath, extractDest); err != nil { - log.Fatal(err) - } - extractDest, err = filepath.Abs(extractDest) - if err != nil { - log.Fatal("error resolving absolute path for protoc", "err", err) - } - return extractDest -} - // protocArchiveBaseName returns the name of the protoc archive file for // the current system, stripped of version and file suffix. func protocArchiveBaseName() (string, error) { @@ -420,12 +467,44 @@ func protocArchiveBaseName() (string, error) { } } +// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc +// in the generate command. It returns the full path of the directory +// containing the 'protoc-gen-go' executable. +func downloadProtocGenGo(cachedir string) string { + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("protoc-gen-go") + if err != nil { + log.Fatal(err) + } + baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH) + archiveName := baseName + if runtime.GOOS == "windows" { + archiveName += ".zip" + } else { + archiveName += ".tar.gz" + } + + archivePath := path.Join(cachedir, archiveName) + if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil { + log.Fatal(err) + } + extractDest := filepath.Join(cachedir, baseName) + if err := build.ExtractArchive(archivePath, extractDest); err != nil { + log.Fatal(err) + } + extractDest, err = filepath.Abs(extractDest) + if err != nil { + log.Fatal("error resolving absolute path for protoc", "err", err) + } + return extractDest +} + // downloadProtoc downloads the prebuilt protoc binary used to lint generated // files as a CI step. It returns the full path to the directory containing // the protoc executable. func downloadProtoc(cachedir string) string { - csdb := build.MustLoadChecksums("build/checksums.txt") - version, err := build.Version(csdb, "protoc") + csdb := download.MustLoadChecksums("build/checksums.txt") + version, err := csdb.FindVersion("protoc") if err != nil { log.Fatal(err) } @@ -436,10 +515,8 @@ func downloadProtoc(cachedir string) string { fileName := fmt.Sprintf("protoc-%s-%s", version, baseName) archiveFileName := fileName + ".zip" - url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName) archivePath := filepath.Join(cachedir, archiveFileName) - - if err := csdb.DownloadFile(url, archivePath); err != nil { + if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil { log.Fatal(err) } extractDest := filepath.Join(cachedir, fileName) @@ -462,11 +539,12 @@ func doXgo(cmdline []string) { env := build.Env() // Make sure xgo is available for cross compilation - gogetxgo := goTool("get", "github.com/karalabe/xgo") + tc := build.GoToolchain{} + gogetxgo := tc.Go("get", "github.com/karalabe/xgo") build.MustRun(gogetxgo) // If all tools building is requested, build everything the builder wants - args := append(buildFlags(env), flag.Args()...) + args := append(buildFlags(env, false, nil), flag.Args()...) if *alltools { args = append(args, []string{"--dest", GOBIN}...) diff --git a/internal/build/download.go b/internal/build/download.go deleted file mode 100644 index 50268227a5..0000000000 --- a/internal/build/download.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "bufio" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "strings" -) - -// ChecksumDB keeps file checksums. -type ChecksumDB struct { - allChecksums []string -} - -// MustLoadChecksums loads a file containing checksums. -func MustLoadChecksums(file string) *ChecksumDB { - content, err := os.ReadFile(file) - if err != nil { - log.Fatal("can't load checksum file: " + err.Error()) - } - return &ChecksumDB{strings.Split(strings.ReplaceAll(string(content), "\r\n", "\n"), "\n")} -} - -// Verify checks whether the given file is valid according to the checksum database. -func (db *ChecksumDB) Verify(path string) error { - fd, err := os.Open(path) - if err != nil { - return err - } - defer fd.Close() - - h := sha256.New() - if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { - return err - } - fileHash := hex.EncodeToString(h.Sum(nil)) - if !db.findHash(filepath.Base(path), fileHash) { - return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path)) - } - return nil -} - -func (db *ChecksumDB) findHash(basename, hash string) bool { - want := hash + " " + basename - for _, line := range db.allChecksums { - if strings.TrimSpace(line) == want { - return true - } - } - return false -} - -// DownloadFile downloads a file and verifies its checksum. -func (db *ChecksumDB) DownloadFile(url, dstPath string) error { - if err := db.Verify(dstPath); err == nil { - fmt.Printf("%s is up-to-date\n", dstPath) - return nil - } - fmt.Printf("%s is stale\n", dstPath) - fmt.Printf("downloading from %s\n", url) - - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("download error: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: status %d", resp.StatusCode) - } - if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { - return err - } - fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return err - } - dst := newDownloadWriter(fd, resp.ContentLength) - _, err = io.Copy(dst, resp.Body) - dst.Close() - if err != nil { - return err - } - return db.Verify(dstPath) -} - -type downloadWriter struct { - file *os.File - dstBuf *bufio.Writer - size int64 - written int64 - lastpct int64 -} - -func newDownloadWriter(dst *os.File, size int64) *downloadWriter { - return &downloadWriter{ - file: dst, - dstBuf: bufio.NewWriter(dst), - size: size, - } -} - -func (w *downloadWriter) Write(buf []byte) (int, error) { - n, err := w.dstBuf.Write(buf) - - // Report progress. - w.written += int64(n) - pct := w.written * 10 / w.size * 10 - if pct != w.lastpct { - if w.lastpct != 0 { - fmt.Print("...") - } - fmt.Print(pct, "%") - w.lastpct = pct - } - return n, err -} - -func (w *downloadWriter) Close() error { - if w.lastpct > 0 { - fmt.Println() // Finish the progress line. - } - flushErr := w.dstBuf.Flush() - closeErr := w.file.Close() - if flushErr != nil { - return flushErr - } - return closeErr -} diff --git a/internal/build/file.go b/internal/build/file.go index 4525f94776..a379354e47 100644 --- a/internal/build/file.go +++ b/internal/build/file.go @@ -25,32 +25,6 @@ import ( "strings" ) -// FileExist checks if a file exists at path. -func FileExist(path string) bool { - _, err := os.Stat(path) - if err != nil && os.IsNotExist(err) { - return false - } - return true -} - -// HashFiles iterates the provided set of files, computing the hash of each. -func HashFiles(files []string) (map[string][32]byte, error) { - res := make(map[string][32]byte) - for _, filePath := range files { - f, err := os.OpenFile(filePath, os.O_RDONLY, 0666) - if err != nil { - return nil, err - } - hasher := sha256.New() - if _, err := io.Copy(hasher, f); err != nil { - return nil, err - } - res[filePath] = [32]byte(hasher.Sum(nil)) - } - return res, nil -} - // HashFolder iterates all files under the given directory, computing the hash // of each. func HashFolder(folder string, exlude []string) (map[string][32]byte, error) { diff --git a/internal/build/gotool.go b/internal/build/gotool.go index 2a47460418..2c7fad2ce3 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -24,6 +24,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/XinFinOrg/XDPoSChain/internal/download" ) type GoToolchain struct { @@ -84,8 +86,8 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { // DownloadGo downloads the Go binary distribution and unpacks it into a temporary // directory. It returns the GOROOT of the unpacked toolchain. -func DownloadGo(csdb *ChecksumDB) string { - version, err := Version(csdb, "golang") +func DownloadGo(csdb *download.ChecksumDB) string { + version, err := csdb.FindVersion("golang") if err != nil { log.Fatal(err) } @@ -130,51 +132,3 @@ func DownloadGo(csdb *ChecksumDB) string { } return goroot } - -// Version returns the versions defined in the checksumdb. -func Version(csdb *ChecksumDB, version string) (string, error) { - for _, l := range csdb.allChecksums { - if !strings.HasPrefix(l, "# version:") { - continue - } - v := strings.Split(l, ":")[1] - parts := strings.Split(v, " ") - if len(parts) != 2 { - log.Print("Erroneous version-string", "v", l) - continue - } - if parts[0] == version { - return parts[1], nil - } - } - return "", fmt.Errorf("no version found for '%v'", version) -} - -// DownloadAndVerifyChecksums downloads all files and checks that they match -// the checksum given in checksums.txt. -// This task can be used to sanity-check new checksums. -func DownloadAndVerifyChecksums(csdb *ChecksumDB) { - var ( - base = "" - ucache = os.TempDir() - ) - for _, l := range csdb.allChecksums { - if strings.HasPrefix(l, "# https://") { - base = l[2:] - continue - } - if strings.HasPrefix(l, "#") { - continue - } - hashFile := strings.Split(l, " ") - if len(hashFile) != 2 { - continue - } - file := hashFile[1] - url := base + file - dst := filepath.Join(ucache, file) - if err := csdb.DownloadFile(url, dst); err != nil { - log.Print(err) - } - } -} diff --git a/internal/build/util.go b/internal/build/util.go index 7334f409dd..8d36c36961 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -17,16 +17,18 @@ package build import ( + "bufio" "bytes" "flag" "fmt" + "io" "log" "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" + "text/template" "time" ) @@ -35,6 +37,9 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { + if cmd.Dir != "" && cmd.Dir != "." { + fmt.Printf("(in %s) ", cmd.Dir) + } fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr @@ -67,6 +72,13 @@ func MustRunCommand(cmd string, args ...string) { // printed while it runs. This is useful for CI builds where the process will be stopped // when there is no output. func MustRunCommandWithOutput(cmd string, args ...string) { + MustRunWithOutput(exec.Command(cmd, args...)) +} + +// MustRunWithOutput runs the given command, and ensures that some output will be printed +// while it runs. This is useful for CI builds where the process will be stopped when +// there is no output. +func MustRunWithOutput(cmd *exec.Cmd) { interval := time.NewTicker(time.Minute) done := make(chan struct{}) defer interval.Stop() @@ -81,16 +93,7 @@ func MustRunCommandWithOutput(cmd string, args ...string) { } } }() - MustRun(exec.Command(cmd, args...)) -} - -// GOPATH returns the value that the GOPATH environment -// variable should be set to. -func GOPATH() string { - if os.Getenv("GOPATH") == "" { - log.Fatal("GOPATH is not set") - } - return os.Getenv("GOPATH") + MustRun(cmd) } var warnedAboutGit bool @@ -101,13 +104,14 @@ func RunGit(args ...string) string { cmd := exec.Command("git", args...) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr - if err := cmd.Run(); err == exec.ErrNotFound { - if !warnedAboutGit { - log.Println("Warning: can't find 'git' in PATH") - warnedAboutGit = true + if err := cmd.Run(); err != nil { + if e, ok := err.(*exec.Error); ok && e.Err == exec.ErrNotFound { + if !warnedAboutGit { + log.Println("Warning: can't find 'git' in PATH") + warnedAboutGit = true + } + return "" } - return "" - } else if err != nil { log.Fatal(strings.Join(cmd.Args, " "), ": ", err, "\n", stderr.String()) } return strings.TrimSpace(stdout.String()) @@ -122,46 +126,136 @@ func readGitFile(file string) string { return strings.TrimSpace(string(content)) } -// GoTool returns the command that runs a go tool. This uses go from GOROOT instead of PATH -// so that go commands executed by build use the same version of Go as the 'host' that runs -// build code. e.g. -// -// /usr/lib/go-1.11/bin/go run build/ci.go ... -// -// runs using go 1.11 and invokes go 1.11 tools from the same GOROOT. This is also important -// because runtime.Version checks on the host should match the tools that are run. -func GoTool(tool string, args ...string) *exec.Cmd { - args = append([]string{tool}, args...) - return exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) +// Render renders the given template file into outputFile. +func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) { + tpl := template.Must(template.ParseFiles(templateFile)) + render(tpl, outputFile, outputPerm, x) } -// ExpandPackages expands a cmd/go import path pattern, skip vendor -// packages, and skip time-consuming tests according to quick flag. -func ExpandPackages(patterns []string, quick bool) []string { - expand := false - for _, pkg := range patterns { - if strings.Contains(pkg, "...") { - expand = true - } - } - if expand { - cmd := GoTool("list", patterns...) - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatalf("package listing failed: %v\n%s", err, string(out)) - } - var packages []string - for _, line := range strings.Split(string(out), "\n") { - if strings.Contains(line, "/vendor/") { - continue - } - if quick && strings.Contains(line, "/consensus/tests/engine_v2_tests") { - continue - } - packages = append(packages, strings.TrimSpace(line)) - - } - return packages - } - return patterns +// RenderString renders the given template string into outputFile. +func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) { + tpl := template.Must(template.New("").Parse(templateContent)) + render(tpl, outputFile, outputPerm, x) +} + +func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x interface{}) { + if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil { + log.Fatal(err) + } + out, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, outputPerm) + if err != nil { + log.Fatal(err) + } + if err := tpl.Execute(out, x); err != nil { + log.Fatal(err) + } + if err := out.Close(); err != nil { + log.Fatal(err) + } +} + +// UploadSFTP uploads files to a remote host using the sftp command line tool. +// The destination host may be specified either as [user@]host: or as a URI in +// the form sftp://[user@]host[:port]. +func UploadSFTP(identityFile, host, dir string, files []string) error { + sftp := exec.Command("sftp") + sftp.Stderr = os.Stderr + if identityFile != "" { + sftp.Args = append(sftp.Args, "-i", identityFile) + } + sftp.Args = append(sftp.Args, host) + fmt.Println(">>>", printArgs(sftp.Args)) + if *DryRunFlag { + return nil + } + + stdin, err := sftp.StdinPipe() + if err != nil { + return fmt.Errorf("can't create stdin pipe for sftp: %v", err) + } + stdout, err := sftp.StdoutPipe() + if err != nil { + return fmt.Errorf("can't create stdout pipe for sftp: %v", err) + } + if err := sftp.Start(); err != nil { + return err + } + in := io.MultiWriter(stdin, os.Stdout) + for _, f := range files { + fmt.Fprintln(in, "put", f, filepath.Join(dir, filepath.Base(f))) + } + fmt.Fprintln(in, "exit") + // Some issue with the PPA sftp server makes it so the server does not + // respond properly to a 'bye', 'exit' or 'quit' from the client. + // To work around that, we check the output, and when we see the client + // exit command, we do a hard exit. + // See + // https://github.com/kolban-google/sftp-gcs/issues/23 + // https://github.com/mscdex/ssh2/pull/1111 + aborted := false + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + txt := scanner.Text() + fmt.Println(txt) + if txt == "sftp> exit" { + // Give it .5 seconds to exit (server might be fixed), then + // hard kill it from the outside + time.Sleep(500 * time.Millisecond) + aborted = true + sftp.Process.Kill() + } + } + }() + stdin.Close() + err = sftp.Wait() + if aborted { + return nil + } + return err +} + +// FindMainPackages finds all 'main' packages in the given directory and returns their +// package paths. +func FindMainPackages(tc *GoToolchain, pattern string) []string { + list := tc.Go("list", "-f", `{{if eq .Name "main"}}{{.ImportPath}}{{end}}`, pattern) + output, err := list.Output() + if err != nil { + log.Fatal("go list failed:", err) + } + var result []string + for l := range bytes.Lines(output) { + l = bytes.TrimSpace(l) + if len(l) > 0 { + result = append(result, string(l)) + } + } + return result +} + +// FindAllPackages expands a cmd/go import path pattern. +func FindAllPackages(tc *GoToolchain) []string { + list := tc.Go("list", "./...") + output, err := list.Output() + if err != nil { + log.Fatal("go list failed:", err) + } + result := make([]string, 0, len(output)) + for line := range bytes.Lines(output) { + pkg := bytes.TrimSpace(line) + if len(pkg) > 0 { + result = append(result, string(pkg)) + } + } + return result +} + +// GOPATH returns the value that the GOPATH environment +// variable should be set to. +func GOPATH() string { + gopath := os.Getenv("GOPATH") + if gopath == "" { + log.Fatal("GOPATH is not set") + } + return gopath } diff --git a/internal/download/download.go b/internal/download/download.go new file mode 100644 index 0000000000..26c7795ce5 --- /dev/null +++ b/internal/download/download.go @@ -0,0 +1,298 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package download implements checksum-verified file downloads. +package download + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "iter" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" +) + +// ChecksumDB keeps file checksums and tool versions. +type ChecksumDB struct { + hashes []hashEntry + versions []versionEntry +} + +type versionEntry struct { + name string + version string +} + +type hashEntry struct { + hash string + file string + url *url.URL +} + +// MustLoadChecksums loads a file containing checksums. +func MustLoadChecksums(file string) *ChecksumDB { + content, err := os.ReadFile(file) + if err != nil { + panic("can't load checksum file: " + err.Error()) + } + db, err := ParseChecksums(content) + if err != nil { + panic(fmt.Sprintf("invalid checksums in %s: %v", file, err)) + } + return db +} + +// ParseChecksums parses a checksum database. +func ParseChecksums(input []byte) (*ChecksumDB, error) { + var ( + csdb = new(ChecksumDB) + rd = bytes.NewBuffer(input) + lastURL *url.URL + ) + for lineNum := 1; ; lineNum++ { + line, err := rd.ReadString('\n') + if err == io.EOF { + break + } + line = strings.TrimSpace(line) + switch { + case line == "": + // Blank lines are allowed, and they reset the current urlEntry. + lastURL = nil + + case strings.HasPrefix(line, "#"): + // It's a comment. Some comments have special meaning. + content := strings.TrimLeft(line, "# ") + switch { + case strings.HasPrefix(content, "version:"): + // Version comments define the version of a tool. + v := strings.Split(content, ":")[1] + parts := strings.Split(v, " ") + if len(parts) != 2 { + return nil, fmt.Errorf("line %d: invalid version string: %q", lineNum, v) + } + csdb.versions = append(csdb.versions, versionEntry{parts[0], parts[1]}) + + case strings.HasPrefix(content, "https://") || strings.HasPrefix(content, "http://"): + // URL comments define the URL where the following files are found. Here + // we keep track of the last found urlEntry and attach it to each file later. + u, err := url.Parse(content) + if err != nil { + return nil, fmt.Errorf("line %d: invalid URL: %v", lineNum, err) + } + lastURL = u + } + + default: + // It's a file hash entry. + fields := strings.Fields(line) + if len(fields) != 2 { + return nil, fmt.Errorf("line %d: invalid number of space-separated fields (%d)", lineNum, len(fields)) + } + csdb.hashes = append(csdb.hashes, hashEntry{fields[0], fields[1], lastURL}) + } + } + return csdb, nil +} + +// Files returns an iterator over all file names. +func (db *ChecksumDB) Files() iter.Seq[string] { + return func(yield func(string) bool) { + for _, e := range db.hashes { + if !yield(e.file) { + return + } + } + } +} + +// DownloadAndVerifyAll downloads all files and checks that they match the checksum given in +// the database. This task can be used to sanity-check new checksums. +func (db *ChecksumDB) DownloadAndVerifyAll() { + var tmp = os.TempDir() + for _, e := range db.hashes { + if e.url == nil { + fmt.Printf("Skipping verification of %s: no URL defined in checksum database", e.file) + continue + } + url := e.url.JoinPath(e.file).String() + dst := filepath.Join(tmp, e.file) + if err := db.DownloadFile(url, dst); err != nil { + fmt.Println("error:", err) + } + } +} + +// verifyHash checks that the file at 'path' has the expected hash. +func verifyHash(path, expectedHash string) error { + fd, err := os.Open(path) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { + return err + } + fileHash := hex.EncodeToString(h.Sum(nil)) + if fileHash != expectedHash { + return fmt.Errorf("invalid file hash: %s %s", fileHash, filepath.Base(path)) + } + return nil +} + +// DownloadFileFromKnownURL downloads a file from the URL defined in the checksum database. +func (db *ChecksumDB) DownloadFileFromKnownURL(dstPath string) error { + base := filepath.Base(dstPath) + url, err := db.FindURL(base) + if err != nil { + return err + } + return db.DownloadFile(url, dstPath) +} + +// DownloadFile downloads a file and verifies its checksum. +func (db *ChecksumDB) DownloadFile(url, dstPath string) error { + basename := filepath.Base(dstPath) + hash := db.findHash(basename) + if hash == "" { + return fmt.Errorf("no known hash for file %q", basename) + } + // Shortcut if already downloaded. + if verifyHash(dstPath, hash) == nil { + fmt.Printf("%s is up-to-date\n", dstPath) + return nil + } + + fmt.Printf("%s is stale\n", dstPath) + fmt.Printf("downloading from %s\n", url) + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("download error: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + + // Download to a temporary file. + tmpfile := dstPath + ".tmp" + fd, err := os.OpenFile(tmpfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + dst := newDownloadWriter(fd, resp.ContentLength) + _, err = io.Copy(dst, resp.Body) + dst.Close() + if err != nil { + os.Remove(tmpfile) + return err + } + if err := verifyHash(tmpfile, hash); err != nil { + os.Remove(tmpfile) + return err + } + // It's valid, rename to dstPath to complete the download. + return os.Rename(tmpfile, dstPath) +} + +// findHash returns the known hash of a file. +func (db *ChecksumDB) findHash(basename string) string { + for _, e := range db.hashes { + if e.file == basename { + return e.hash + } + } + return "" +} + +// FindVersion returns the current known version of a tool, if it is defined in the file. +func (db *ChecksumDB) FindVersion(tool string) (string, error) { + for _, e := range db.versions { + if e.name == tool { + return e.version, nil + } + } + return "", fmt.Errorf("tool version %q not defined in checksum database", tool) +} + +// FindURL gets the URL for a file. +func (db *ChecksumDB) FindURL(basename string) (string, error) { + for _, e := range db.hashes { + if e.file == basename { + if e.url == nil { + return "", fmt.Errorf("file %q has no URL defined", e.file) + } + return e.url.JoinPath(e.file).String(), nil + } + } + return "", fmt.Errorf("file %q does not exist in checksum database", basename) +} + +type downloadWriter struct { + file *os.File + dstBuf *bufio.Writer + size int64 + written int64 + lastpct int64 +} + +func newDownloadWriter(dst *os.File, size int64) *downloadWriter { + return &downloadWriter{ + file: dst, + dstBuf: bufio.NewWriter(dst), + size: size, + } +} + +func (w *downloadWriter) Write(buf []byte) (int, error) { + n, err := w.dstBuf.Write(buf) + + // Report progress. + w.written += int64(n) + pct := w.written * 10 / w.size * 10 + if pct != w.lastpct { + if w.lastpct != 0 { + fmt.Print("...") + } + fmt.Print(pct, "%") + w.lastpct = pct + } + return n, err +} + +func (w *downloadWriter) Close() error { + if w.lastpct > 0 { + fmt.Println() // Finish the progress line. + } + flushErr := w.dstBuf.Flush() + closeErr := w.file.Close() + if flushErr != nil { + return flushErr + } + return closeErr +}