From 464b42eff6e422d88eb4d4e3a0205d81ad153a82 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 15 Nov 2019 11:39:53 +0100 Subject: [PATCH] build: use golangci-lint This changes build/ci.go to download and run golangci-lint instead of gometalinter. --- .gitignore | 1 + .golangci.yml | 40 +++++++++++ .travis.yml | 7 +- build/checksums.txt | 19 +++++ build/ci.go | 78 +++++++++++---------- internal/build/archive.go | 5 +- internal/build/download.go | 140 +++++++++++++++++++++++++++++++++++++ internal/build/gosrc.go | 81 --------------------- 8 files changed, 250 insertions(+), 121 deletions(-) create mode 100644 .golangci.yml create mode 100644 build/checksums.txt create mode 100644 internal/build/download.go delete mode 100644 internal/build/gosrc.go diff --git a/.gitignore b/.gitignore index 3016facee0..1ee8b83022 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ build/_vendor/pkg # used by the Makefile /build/_workspace/ +/build/cache/ /build/bin/ /geth*.zip diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..16af0fad54 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,40 @@ +# This file configures github.com/golangci/golangci-lint. + +run: + timeout: 2m + tests: true + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + +linters: + disable-all: true + enable: + - deadcode + - goimports + - gosimple + - govet + - misspell + # - staticcheck + - unconvert + # - unused + - varcheck + +linters-settings: + gofmt: + simplify: true + goconst: + min-len: 3 # minimum length of string constant + min-occurrences: 6 # minimum number of occurrences + +issues: + exclude-rules: + - path: crypto/blake2b/ + linters: + - deadcode + - path: crypto/bn256/cloudflare + linters: + - deadcode + - path: p2p/discv5/ + linters: + - deadcode diff --git a/.travis.yml b/.travis.yml index c992451c4f..9775005976 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ jobs: - lint git: submodules: false # avoid cloning ethereum/tests + cache: + directories: + - ./build/cache/ script: - go run build/ci.go lint @@ -87,10 +90,10 @@ jobs: - python-paramiko cache: directories: - - $HOME/.gobundle + - ./build/cache/ script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz + - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - stage: build diff --git a/build/checksums.txt b/build/checksums.txt new file mode 100644 index 0000000000..bb814e3392 --- /dev/null +++ b/build/checksums.txt @@ -0,0 +1,19 @@ +# This file contains sha256 checksums of optional build dependencies. + +95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 go1.13.4.src.tar.gz + +1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb golangci-lint-1.21.0-linux-386.tar.gz +267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90 golangci-lint-1.21.0-linux-armv6.tar.gz +a602c1f25f90e46e621019cff0a8cb3f4e1837011f3537f15e730d6a9ebf507b golangci-lint-1.21.0-freebsd-armv7.tar.gz +2c861f8dc56b560474aa27cab0c075991628cc01af3451e27ac82f5d10d5106b golangci-lint-1.21.0-linux-amd64.tar.gz +a1c39e055280e755acaa906e7abfc20b99a5c28be8af541c57fbc44abbb20dde golangci-lint-1.21.0-linux-arm64.tar.gz +a8f8bda8c6a4136acf858091077830b1e83ad5612606cb69d5dced869ce00bd8 golangci-lint-1.21.0-linux-ppc64le.tar.gz +0a8a8c3bc660ccbca668897ab520f7ee9878f16cc8e4dd24fe46236ceec97ba3 golangci-lint-1.21.0-freebsd-armv6.tar.gz +699b07f45e216571f54002bcbd83b511c4801464a422162158e299587b095b18 golangci-lint-1.21.0-freebsd-amd64.tar.gz +980fb4993942154bb5c8129ea3b86de09574fe81b24384ebb58cd7a9d2f04483 golangci-lint-1.21.0-linux-armv7.tar.gz +f15b689088a47f20d5d3c1d945e9ee7c6238f2b84ea468b5f886cf8713dce62e golangci-lint-1.21.0-windows-386.zip +2e40ded7adcf11e59013cb15c24438b15a86526ca241edfcfdf1abd73a5280a8 golangci-lint-1.21.0-windows-amd64.zip +6052c7cfea4d6dc2fc722f6c12792a5ec087420198db495afffbc22052653bf7 golangci-lint-1.21.0-freebsd-386.tar.gz +ca00b8eacf9af14a71b908b4149606c762aa5c0eac781e74ca0abedfdfdf6c8c golangci-lint-1.21.0-linux-s390x.tar.gz +1365455940c342f95718159d89d66ad2eef19f0846c3e87023e915a3527b929f golangci-lint-1.21.0-darwin-386.tar.gz +2b2713ec5007e67883aa501eebb81f22abfab0cf0909134ba90f60a066db3760 golangci-lint-1.21.0-darwin-amd64.tar.gz diff --git a/build/ci.go b/build/ci.go index ac5c72b6b1..d41a415fb9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -58,7 +58,6 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/build" "github.com/ethereum/go-ethereum/params" ) @@ -336,39 +335,37 @@ func doTest(cmdline []string) { build.MustRun(gotest) } -// runs gometalinter on requested packages +// doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { + var ( + cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") + ) flag.CommandLine.Parse(cmdline) - packages := []string{"./..."} if len(flag.CommandLine.Args()) > 0 { packages = flag.CommandLine.Args() } - // Get metalinter and install all supported linters - build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") - // Run fast linters batched together - configs := []string{ - "--vendor", - "--tests", - "--deadline=2m", - "--disable-all", - "--enable=goimports", - "--enable=varcheck", - "--enable=vet", - "--enable=gofmt", - "--enable=misspell", - "--enable=goconst", - "--min-occurrences=6", // for goconst - } - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) + linter := downloadLinter(*cachedir) + lflags := []string{"run", "--config", ".golangci.yml"} + build.MustRunCommand(linter, append(lflags, packages...)...) +} - // Run slow linters one by one - for _, linter := range []string{"unconvert", "gosimple"} { - configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) +// downloadLinter downloads and unpacks golangci-lint. +func downloadLinter(cachedir string) string { + const version = "1.21.0" + + csdb := build.MustLoadChecksums("build/checksums.txt") + base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) + url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) + archivePath := filepath.Join(cachedir, base+".tar.gz") + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) } + if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil { + log.Fatal(err) + } + return filepath.Join(cachedir, base, "golangci-lint") } // Release Packaging @@ -472,8 +469,7 @@ func maybeSkipArchive(env build.Environment) { func doDebianSource(cmdline []string) { var ( goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`) - gobundle = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`) - gohash = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`) + cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) signer = flag.String("signer", "", `Signing key name, also used as package author`) upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) @@ -491,24 +487,25 @@ func doDebianSource(cmdline []string) { gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) } - // Download and verify the Go source package - if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil { - log.Fatalf("Failed to ensure Go source package: %v", err) - } - // Create Debian packages and upload them + + // Download and verify the Go source package. + gobundle := downloadGoSources(*goversion, *cachedir) + + // Create Debian packages and upload them. for _, pkg := range debPackages { for distro, goboot := range debDistroGoBoots { - // Prepare the debian package with the go-ethereum sources + // Prepare the debian package with the go-ethereum sources. meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) - // Ship the Go sources along so we have a proper thing to build with - if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil { + // Add Go source code. + if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil { log.Fatalf("Failed to extract Go sources: %v", err) } if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { log.Fatalf("Failed to rename Go source folder: %v", err) } + // Run the packaging and upload to the PPA debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") debuild.Dir = pkgdir @@ -530,6 +527,17 @@ func doDebianSource(cmdline []string) { } } +func downloadGoSources(version string, cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + file := fmt.Sprintf("go%s.src.tar.gz", version) + url := "https://dl.google.com/go/" + file + dst := filepath.Join(cachedir, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Fatal(err) + } + return dst +} + func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { diff --git a/internal/build/archive.go b/internal/build/archive.go index 8571edd5a5..a00258d999 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -213,11 +213,10 @@ func ExtractTarballArchive(archive string, dest string) error { target := filepath.Join(dest, header.Name) switch header.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(target, 0755); err != nil { + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err } - case tar.TypeReg: file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { return err diff --git a/internal/build/download.go b/internal/build/download.go new file mode 100644 index 0000000000..97ecf373ed --- /dev/null +++ b/internal/build/download.go @@ -0,0 +1,140 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package build + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// ChecksumDB keeps file checksums. +type ChecksumDB struct { + allChecksums []string +} + +// MustLoadChecksums loads a file containing checksums. +func MustLoadChecksums(file string) *ChecksumDB { + content, err := ioutil.ReadFile(file) + if err != nil { + log.Fatal("can't load checksum file: " + err.Error()) + } + return &ChecksumDB{strings.Split(string(content), "\n")} +} + +// Verify checks whether the given file is valid according to the checksum database. +func (db *ChecksumDB) Verify(path string) error { + fd, err := os.Open(path) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { + return err + } + fileHash := hex.EncodeToString(h.Sum(nil)) + if !db.findHash(filepath.Base(path), fileHash) { + return fmt.Errorf("invalid file hash %s", fileHash) + } + return nil +} + +func (db *ChecksumDB) findHash(basename, hash string) bool { + want := hash + " " + basename + for _, line := range db.allChecksums { + if strings.TrimSpace(line) == want { + return true + } + } + return false +} + +// DownloadFile downloads a file and verifies its checksum. +func (db *ChecksumDB) DownloadFile(url, dstPath string) error { + if err := db.Verify(dstPath); err == nil { + fmt.Printf("%s is up-to-date\n", dstPath) + return nil + } + fmt.Printf("%s is stale\n", dstPath) + fmt.Printf("downloading from %s\n", url) + + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: code %d, err %v", resp.StatusCode, err) + } + defer resp.Body.Close() + + fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + dst := bufio.NewWriter(io.MultiWriter(fd, &dots{length: resp.ContentLength})) + _, copyErr := io.Copy(dst, resp.Body) + flushErr := dst.Flush() + fd.Close() + if copyErr != nil { + return copyErr + } else if flushErr != nil { + return flushErr + } + + return db.Verify(dstPath) +} + +type dots struct { + c int64 + length int64 + lastpct int64 +} + +func (d *dots) Write(buf []byte) (int, error) { + d.c += int64(len(buf)) + pct := d.c * 10 / d.length * 10 + if pct != d.lastpct { + if d.lastpct != 0 { + fmt.Print("...") + } + fmt.Print(pct, "%") + d.lastpct = pct + } + if pct == 100 { + fmt.Println() + } + return len(buf), nil +} diff --git a/internal/build/gosrc.go b/internal/build/gosrc.go deleted file mode 100644 index c85e469680..0000000000 --- a/internal/build/gosrc.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "bytes" - "crypto/sha256" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" -) - -// EnsureGoSources ensures that path contains a file with the given SHA256 hash, -// and if not, it downloads a fresh Go source package from upstream and replaces -// path with it (if the hash matches). -func EnsureGoSources(version string, hash []byte, path string) error { - // Sanity check the destination path to ensure we don't do weird things - if !strings.HasSuffix(path, ".tar.gz") { - return fmt.Errorf("destination path (%s) must end with .tar.gz", path) - } - // If the file exists, validate it's hash - if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all - hasher := sha256.New() - hasher.Write(archive) - have := hasher.Sum(nil) - - if bytes.Equal(have, hash) { - fmt.Printf("Go %s [%x] available at %s\n", version, hash, path) - return nil - } - fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path) - if err := os.Remove(path); err != nil { - return err - } - } - // Archive missing or bad hash, download a new one - fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path) - - res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version)) - if err != nil || res.StatusCode != http.StatusOK { - return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err) - } - defer res.Body.Close() - - archive, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - // Sanity check the downloaded archive, save if checks out - hasher := sha256.New() - hasher.Write(archive) - - if have := hasher.Sum(nil); !bytes.Equal(have, hash) { - return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash) - } - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return err - } - if err := ioutil.WriteFile(path, archive, 0644); err != nil { - return err - } - fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path) - return nil -}