diff --git a/build/ci.go b/build/ci.go index cec9b00a0a..6ed0bf7f6f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -58,6 +58,7 @@ import ( "strings" "time" + "github.com/cespare/cp" "github.com/maticnetwork/bor/internal/build" "github.com/maticnetwork/bor/params" ) @@ -137,7 +138,20 @@ var ( // Note: yakkety is unsupported because it was officially deprecated on Launchpad. // Note: zesty is unsupported because it was officially deprecated on Launchpad. // Note: artful is unsupported because it was officially deprecated on Launchpad. - debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"} + // Note: cosmic is unsupported because it was officially deprecated on Launchpad. + debDistroGoBoots = map[string]string{ + "trusty": "golang-1.11", + "xenial": "golang-go", + "bionic": "golang-go", + "disco": "golang-go", + "eoan": "golang-go", + "focal": "golang-go", + } + + debGoBootPaths = map[string]string{ + "golang-1.11": "/usr/lib/go-1.11", + "golang-go": "/usr/lib/go", + } ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -201,9 +215,9 @@ func doInstall(cmdline []string) { var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 9 { + if minor < 11 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.9 and cannot") + log.Println("go-ethereum requires at least Go version 1.11 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } @@ -213,22 +227,18 @@ func doInstall(cmdline []string) { if flag.NArg() > 0 { packages = flag.Args() } - packages = build.ExpandPackagesNoVendor(packages) if *arch == "" || *arch == runtime.GOARCH { goinstall := goTool("install", buildFlags(env)...) + if runtime.GOARCH == "arm64" { + goinstall.Args = append(goinstall.Args, "-p", "1") + } goinstall.Args = append(goinstall.Args, "-v") goinstall.Args = append(goinstall.Args, packages...) build.MustRun(goinstall) return } - // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds - if *arch == "arm" { - os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) - for _, path := range filepath.SplitList(build.GOPATH()) { - os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) - } - } + // Seems we are cross compiling, work around forbidden GOBIN goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) goinstall.Args = append(goinstall.Args, "-v") @@ -278,7 +288,6 @@ func goTool(subcmd string, args ...string) *exec.Cmd { func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { cmd := build.GoTool(subcmd, args...) - cmd.Env = []string{"GOPATH=" + build.GOPATH()} if arch == "" || arch == runtime.GOARCH { cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) } else { @@ -289,7 +298,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd cmd.Env = append(cmd.Env, "CC="+cc) } for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { + if strings.HasPrefix(e, "GOBIN=") { continue } cmd.Env = append(cmd.Env, e) @@ -303,6 +312,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd func doTest(cmdline []string) { coverage := flag.Bool("coverage", false, "Whether to record code coverage") + verbose := flag.Bool("v", false, "Whether to log verbosely") flag.CommandLine.Parse(cmdline) env := build.Env() @@ -310,7 +320,6 @@ func doTest(cmdline []string) { if len(flag.CommandLine.Args()) > 0 { packages = flag.CommandLine.Args() } - packages = build.ExpandPackagesNoVendor(packages) // Run the actual tests. // Test a single package at a time. CI builders are slow @@ -320,44 +329,46 @@ func doTest(cmdline []string) { if *coverage { gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") } + if *verbose { + gotest.Args = append(gotest.Args, "-v") + } gotest.Args = append(gotest.Args, packages...) build.MustRun(gotest) } -// runs gometalinter on requested packages +// doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { + var ( + cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") + ) flag.CommandLine.Parse(cmdline) - packages := []string{"./..."} if len(flag.CommandLine.Args()) > 0 { packages = flag.CommandLine.Args() } - // Get metalinter and install all supported linters - build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") - // Run fast linters batched together - configs := []string{ - "--vendor", - "--tests", - "--deadline=2m", - "--disable-all", - "--enable=goimports", - "--enable=varcheck", - "--enable=vet", - "--enable=gofmt", - "--enable=misspell", - "--enable=goconst", - "--min-occurrences=6", // for goconst - } - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) + linter := downloadLinter(*cachedir) + lflags := []string{"run", "--config", ".golangci.yml"} + build.MustRunCommand(linter, append(lflags, packages...)...) + fmt.Println("You have achieved perfection.") +} - // Run slow linters one by one - for _, linter := range []string{"unconvert", "gosimple"} { - configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) +// downloadLinter downloads and unpacks golangci-lint. +func downloadLinter(cachedir string) string { + const version = "1.22.2" + + csdb := build.MustLoadChecksums("build/checksums.txt") + base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) + url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) + archivePath := filepath.Join(cachedir, base+".tar.gz") + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) } + if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil { + log.Fatal(err) + } + return filepath.Join(cachedir, base, "golangci-lint") } // Release Packaging @@ -460,11 +471,13 @@ func maybeSkipArchive(env build.Environment) { // Debian Packaging func doDebianSource(cmdline []string) { var ( - signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) - sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) - workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) - now = time.Now() + goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`) + cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) + signer = flag.String("signer", "", `Signing key name, also used as package author`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) + sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) + workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) + now = time.Now() ) flag.CommandLine.Parse(cmdline) *workdir = makeWorkdir(*workdir) @@ -478,12 +491,39 @@ func doDebianSource(cmdline []string) { build.MustRun(gpg) } - // Create Debian packages and upload them + // Download and verify the Go source package. + gobundle := downloadGoSources(*goversion, *cachedir) + + // Download all the dependencies needed to build the sources and run the ci script + srcdepfetch := goTool("install", "-n", "./...") + srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) + build.MustRun(srcdepfetch) + + cidepfetch := goTool("run", "./build/ci.go") + cidepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath")) + cidepfetch.Run() // Command fails, don't care, we only need the deps to start it + + // Create Debian packages and upload them. for _, pkg := range debPackages { - for _, distro := range debDistros { - meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) + for distro, goboot := range debDistroGoBoots { + // Prepare the debian package with the go-ethereum sources. + meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) - debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") + + // Add Go source code + if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil { + log.Fatalf("Failed to extract Go sources: %v", err) + } + if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { + log.Fatalf("Failed to rename Go source folder: %v", err) + } + // Add all dependency modules in compressed form + os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) + if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { + log.Fatalf("Failed to copy Go module dependencies: %v", err) + } + // Run the packaging and upload to the PPA + debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") debuild.Dir = pkgdir build.MustRun(debuild) @@ -503,6 +543,17 @@ func doDebianSource(cmdline []string) { } } +func downloadGoSources(version string, cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + file := fmt.Sprintf("go%s.src.tar.gz", version) + url := "https://dl.google.com/go/" + file + dst := filepath.Join(cachedir, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Fatal(err) + } + return dst +} + func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { @@ -562,7 +613,9 @@ type debPackage struct { } type debMetadata struct { - Env build.Environment + Env build.Environment + GoBootPackage string + GoBootPath string PackageName string @@ -591,19 +644,21 @@ func (d debExecutable) Package() string { return d.BinaryName } -func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { +func newDebMetadata(distro, goboot, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { if author == "" { // No signing key, use default author. author = "Ethereum Builds " } return debMetadata{ - PackageName: name, - Env: env, - Author: author, - Distro: distro, - Version: version, - Time: t.Format(time.RFC1123Z), - Executables: exes, + GoBootPackage: goboot, + GoBootPath: debGoBootPaths[goboot], + PackageName: name, + Env: env, + Author: author, + Distro: distro, + Version: version, + Time: t.Format(time.RFC1123Z), + Executables: exes, } } @@ -668,7 +723,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { if err := os.Mkdir(pkgdir, 0755); err != nil { log.Fatal(err) } - // Copy the source code. build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) @@ -686,7 +740,6 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe) build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe) } - return pkgdir } @@ -715,7 +768,7 @@ func doWindowsInstaller(cmdline []string) { continue } allTools = append(allTools, filepath.Base(file)) - if filepath.Base(file) == "geth.exe" { + if filepath.Base(file) == "bor.exe" { gethTool = file } else { devTools = append(devTools, file) @@ -729,14 +782,17 @@ func doWindowsInstaller(cmdline []string) { "Geth": gethTool, "DevTools": devTools, } - build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) + build.Render("build/nsis.bor.nsi", filepath.Join(*workdir, "bor.nsi"), 0644, nil) build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) - build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755) - build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755) - + if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { + log.Fatal("Failed to copy SimpleFC.dll: %v", err) + } + if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { + log.Fatal("Failed to copy copyright note: %v", err) + } // Build the installer. This assumes that all the needed files have been previously // built (don't mix building and packaging to keep cross compilation complexity to a // minimum). @@ -744,16 +800,15 @@ func doWindowsInstaller(cmdline []string) { if env.Commit != "" { version[2] += "-" + env.Commit[:8] } - installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") + installer, _ := filepath.Abs("bor-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") build.MustRunCommand("makensis.exe", "/DOUTPUTFILE="+installer, "/DMAJORVERSION="+version[0], "/DMINORVERSION="+version[1], "/DBUILDVERSION="+version[2], "/DARCH="+*arch, - filepath.Join(*workdir, "geth.nsi"), + filepath.Join(*workdir, "bor.nsi"), ) - // Sign and publish installer. if err := archiveUpload(installer, *upload, *signer); err != nil { log.Fatal(err) @@ -826,7 +881,6 @@ func gomobileTool(subcmd string, args ...string) *exec.Cmd { cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) cmd.Args = append(cmd.Args, args...) cmd.Env = []string{ - "GOPATH=" + build.GOPATH(), "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), } for _, e := range os.Environ() { @@ -899,7 +953,7 @@ func doXCodeFramework(cmdline []string) { // Build the iOS XCode framework build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init")) - bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/maticnetwork/bor/mobile") + bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/maticnetwork/bor/mobile") if *local { // If we're building locally, use the build folder and stop afterwards @@ -907,7 +961,7 @@ func doXCodeFramework(cmdline []string) { build.MustRun(bind) return } - archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) + archive := "bor-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) if err := os.Mkdir(archive, os.ModePerm); err != nil { log.Fatal(err) } @@ -1014,16 +1068,10 @@ func doXgo(cmdline []string) { func xgoTool(args []string) *exec.Cmd { cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) - cmd.Env = []string{ - "GOPATH=" + build.GOPATH(), + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, []string{ "GOBIN=" + GOBIN, - } - for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { - continue - } - cmd.Env = append(cmd.Env, e) - } + }...) return cmd } diff --git a/internal/build/archive.go b/internal/build/archive.go index ac680ba63d..a00258d999 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -183,3 +183,48 @@ func (a *TarballArchive) Close() error { } return a.file.Close() } + +func ExtractTarballArchive(archive string, dest string) error { + // We're only interested in gzipped archives, wrap the reader now + ar, err := os.Open(archive) + if err != nil { + return err + } + defer ar.Close() + + gzr, err := gzip.NewReader(ar) + if err != nil { + return err + } + defer gzr.Close() + + // Iterate over all the files in the tarball + tr := tar.NewReader(gzr) + for { + // Fetch the next tarball header and abort if needed + header, err := tr.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + // Figure out the target and create it + target := filepath.Join(dest, header.Name) + + switch header.Typeflag { + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(file, tr); err != nil { + return err + } + file.Close() + } + } +} diff --git a/internal/build/download.go b/internal/build/download.go new file mode 100644 index 0000000000..0ed0b5e130 --- /dev/null +++ b/internal/build/download.go @@ -0,0 +1,151 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package build + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +// ChecksumDB keeps file checksums. +type ChecksumDB struct { + allChecksums []string +} + +// MustLoadChecksums loads a file containing checksums. +func MustLoadChecksums(file string) *ChecksumDB { + content, err := ioutil.ReadFile(file) + if err != nil { + log.Fatal("can't load checksum file: " + err.Error()) + } + return &ChecksumDB{strings.Split(string(content), "\n")} +} + +// Verify checks whether the given file is valid according to the checksum database. +func (db *ChecksumDB) Verify(path string) error { + fd, err := os.Open(path) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { + return err + } + fileHash := hex.EncodeToString(h.Sum(nil)) + if !db.findHash(filepath.Base(path), fileHash) { + return fmt.Errorf("invalid file hash %s", fileHash) + } + return nil +} + +func (db *ChecksumDB) findHash(basename, hash string) bool { + want := hash + " " + basename + for _, line := range db.allChecksums { + if strings.TrimSpace(line) == want { + return true + } + } + return false +} + +// DownloadFile downloads a file and verifies its checksum. +func (db *ChecksumDB) DownloadFile(url, dstPath string) error { + if err := db.Verify(dstPath); err == nil { + fmt.Printf("%s is up-to-date\n", dstPath) + return nil + } + fmt.Printf("%s is stale\n", dstPath) + fmt.Printf("downloading from %s\n", url) + + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("download error: %v", err) + } else if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) + } + defer resp.Body.Close() + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + dst := newDownloadWriter(fd, resp.ContentLength) + _, err = io.Copy(dst, resp.Body) + dst.Close() + if err != nil { + return err + } + + return db.Verify(dstPath) +} + +type downloadWriter struct { + file *os.File + dstBuf *bufio.Writer + size int64 + written int64 + lastpct int64 +} + +func newDownloadWriter(dst *os.File, size int64) *downloadWriter { + return &downloadWriter{ + file: dst, + dstBuf: bufio.NewWriter(dst), + size: size, + } +} + +func (w *downloadWriter) Write(buf []byte) (int, error) { + n, err := w.dstBuf.Write(buf) + + // Report progress. + w.written += int64(n) + pct := w.written * 10 / w.size * 10 + if pct != w.lastpct { + if w.lastpct != 0 { + fmt.Print("...") + } + fmt.Print(pct, "%") + w.lastpct = pct + } + return n, err +} + +func (w *downloadWriter) Close() error { + if w.lastpct > 0 { + fmt.Println() // Finish the progress line. + } + flushErr := w.dstBuf.Flush() + closeErr := w.file.Close() + if flushErr != nil { + return flushErr + } + return closeErr +}