build: use golangci-lint

This changes build/ci.go to download and run golangci-lint instead
of gometalinter.

.gitignore

@@ -24,6 +24,7 @@ build/_vendor/pkg
 
 # used by the Makefile
 /build/_workspace/
+/build/cache/
 /build/bin/
 /geth*.zip
 

.golangci.yml

@@ -0,0 +1,40 @@
+# This file configures github.com/golangci/golangci-lint.
+
+run:
+  timeout: 2m
+  tests: true
+  # default is true. Enables skipping of directories:
+  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs-use-default: true
+
+linters:
+  disable-all: true
+  enable:
+    - deadcode
+    - goimports
+    - gosimple
+    - govet
+    - misspell
+    # - staticcheck
+    - unconvert
+    # - unused
+    - varcheck
+
+linters-settings:
+  gofmt:
+    simplify: true
+  goconst:
+    min-len: 3 # minimum length of string constant
+    min-occurrences: 6 # minimum number of occurrences
+
+issues:
+  exclude-rules:
+    - path: crypto/blake2b/
+      linters:
+        - deadcode
+    - path: crypto/bn256/cloudflare
+      linters:
+        - deadcode
+    - path: p2p/discv5/
+      linters:
+        - deadcode

.travis.yml

@@ -12,6 +12,9 @@ jobs:
         - lint
       git:
         submodules: false # avoid cloning ethereum/tests
+      cache:
+        directories:
+          - ./build/cache/
       script:
         - go run build/ci.go lint
 
@@ -87,10 +90,10 @@ jobs:
             - python-paramiko
       cache:
         directories:
-          - $HOME/.gobundle
+          - ./build/cache/
       script:
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz
+        - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - stage: build

build/checksums.txt

@@ -0,0 +1,19 @@
+# This file contains sha256 checksums of optional build dependencies.
+
+95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624  go1.13.4.src.tar.gz
+
+1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb  golangci-lint-1.21.0-linux-386.tar.gz
+267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90  golangci-lint-1.21.0-linux-armv6.tar.gz
+a602c1f25f90e46e621019cff0a8cb3f4e1837011f3537f15e730d6a9ebf507b  golangci-lint-1.21.0-freebsd-armv7.tar.gz
+2c861f8dc56b560474aa27cab0c075991628cc01af3451e27ac82f5d10d5106b  golangci-lint-1.21.0-linux-amd64.tar.gz
+a1c39e055280e755acaa906e7abfc20b99a5c28be8af541c57fbc44abbb20dde  golangci-lint-1.21.0-linux-arm64.tar.gz
+a8f8bda8c6a4136acf858091077830b1e83ad5612606cb69d5dced869ce00bd8  golangci-lint-1.21.0-linux-ppc64le.tar.gz
+0a8a8c3bc660ccbca668897ab520f7ee9878f16cc8e4dd24fe46236ceec97ba3  golangci-lint-1.21.0-freebsd-armv6.tar.gz
+699b07f45e216571f54002bcbd83b511c4801464a422162158e299587b095b18  golangci-lint-1.21.0-freebsd-amd64.tar.gz
+980fb4993942154bb5c8129ea3b86de09574fe81b24384ebb58cd7a9d2f04483  golangci-lint-1.21.0-linux-armv7.tar.gz
+f15b689088a47f20d5d3c1d945e9ee7c6238f2b84ea468b5f886cf8713dce62e  golangci-lint-1.21.0-windows-386.zip
+2e40ded7adcf11e59013cb15c24438b15a86526ca241edfcfdf1abd73a5280a8  golangci-lint-1.21.0-windows-amd64.zip
+6052c7cfea4d6dc2fc722f6c12792a5ec087420198db495afffbc22052653bf7  golangci-lint-1.21.0-freebsd-386.tar.gz
+ca00b8eacf9af14a71b908b4149606c762aa5c0eac781e74ca0abedfdfdf6c8c  golangci-lint-1.21.0-linux-s390x.tar.gz
+1365455940c342f95718159d89d66ad2eef19f0846c3e87023e915a3527b929f  golangci-lint-1.21.0-darwin-386.tar.gz
+2b2713ec5007e67883aa501eebb81f22abfab0cf0909134ba90f60a066db3760  golangci-lint-1.21.0-darwin-amd64.tar.gz

build/ci.go

@@ -58,7 +58,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/build"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -336,39 +335,37 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
-// runs gometalinter on requested packages
+// doLint runs golangci-lint on requested packages.
 func doLint(cmdline []string) {
+	var (
+		cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.")
+	)
 	flag.CommandLine.Parse(cmdline)
-
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
 		packages = flag.CommandLine.Args()
 	}
-	// Get metalinter and install all supported linters
-	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
 
-	// Run fast linters batched together
+	linter := downloadLinter(*cachedir)
-	configs := []string{
+	lflags := []string{"run", "--config", ".golangci.yml"}
-		"--vendor",
+	build.MustRunCommand(linter, append(lflags, packages...)...)
-		"--tests",
+}
-		"--deadline=2m",
-		"--disable-all",
-		"--enable=goimports",
-		"--enable=varcheck",
-		"--enable=vet",
-		"--enable=gofmt",
-		"--enable=misspell",
-		"--enable=goconst",
-		"--min-occurrences=6", // for goconst
-	}
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 
-	// Run slow linters one by one
+// downloadLinter downloads and unpacks golangci-lint.
-	for _, linter := range []string{"unconvert", "gosimple"} {
+func downloadLinter(cachedir string) string {
-		configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
+	const version = "1.21.0"
-		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
+
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)
+	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base)
+	archivePath := filepath.Join(cachedir, base+".tar.gz")
+	if err := csdb.DownloadFile(url, archivePath); err != nil {
+		log.Fatal(err)
 	}
+	if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil {
+		log.Fatal(err)
+	}
+	return filepath.Join(cachedir, base, "golangci-lint")
 }
 
 // Release Packaging
@@ -472,8 +469,7 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
-		gobundle  = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`)
+		cachedir  = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
-		gohash    = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`)
 		signer    = flag.String("signer", "", `Signing key name, also used as package author`)
 		upload    = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
 		sshUser   = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
@@ -491,24 +487,25 @@ func doDebianSource(cmdline []string) {
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
 	}
-	// Download and verify the Go source package
+
-	if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil {
+	// Download and verify the Go source package.
-		log.Fatalf("Failed to ensure Go source package: %v", err)
+	gobundle := downloadGoSources(*goversion, *cachedir)
-	}
+
-	// Create Debian packages and upload them
+	// Create Debian packages and upload them.
 	for _, pkg := range debPackages {
 		for distro, goboot := range debDistroGoBoots {
-			// Prepare the debian package with the go-ethereum sources
+			// Prepare the debian package with the go-ethereum sources.
 			meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
 
-			// Ship the Go sources along so we have a proper thing to build with
+			// Add Go source code.
-			if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil {
+			if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil {
 				log.Fatalf("Failed to extract Go sources: %v", err)
 			}
 			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
 				log.Fatalf("Failed to rename Go source folder: %v", err)
 			}
+
 			// Run the packaging and upload to the PPA
 			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
@@ -530,6 +527,17 @@ func doDebianSource(cmdline []string) {
 	}
 }
 
+func downloadGoSources(version string, cachedir string) string {
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	file := fmt.Sprintf("go%s.src.tar.gz", version)
+	url := "https://dl.google.com/go/" + file
+	dst := filepath.Join(cachedir, file)
+	if err := csdb.DownloadFile(url, dst); err != nil {
+		log.Fatal(err)
+	}
+	return dst
+}
+
 func ppaUpload(workdir, ppa, sshUser string, files []string) {
 	p := strings.Split(ppa, "/")
 	if len(p) != 2 {

internal/build/archive.go

@@ -213,11 +213,10 @@ func ExtractTarballArchive(archive string, dest string) error {
 		target := filepath.Join(dest, header.Name)
 
 		switch header.Typeflag {
-		case tar.TypeDir:
+		case tar.TypeReg:
-			if err := os.MkdirAll(target, 0755); err != nil {
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
 				return err
 			}
-		case tar.TypeReg:
 			file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
 			if err != nil {
 				return err

internal/build/download.go

@@ -0,0 +1,140 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"bufio"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// ChecksumDB keeps file checksums.
+type ChecksumDB struct {
+	allChecksums []string
+}
+
+// MustLoadChecksums loads a file containing checksums.
+func MustLoadChecksums(file string) *ChecksumDB {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatal("can't load checksum file: " + err.Error())
+	}
+	return &ChecksumDB{strings.Split(string(content), "\n")}
+}
+
+// Verify checks whether the given file is valid according to the checksum database.
+func (db *ChecksumDB) Verify(path string) error {
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
+		return err
+	}
+	fileHash := hex.EncodeToString(h.Sum(nil))
+	if !db.findHash(filepath.Base(path), fileHash) {
+		return fmt.Errorf("invalid file hash %s", fileHash)
+	}
+	return nil
+}
+
+func (db *ChecksumDB) findHash(basename, hash string) bool {
+	want := hash + "  " + basename
+	for _, line := range db.allChecksums {
+		if strings.TrimSpace(line) == want {
+			return true
+		}
+	}
+	return false
+}
+
+// DownloadFile downloads a file and verifies its checksum.
+func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
+	if err := db.Verify(dstPath); err == nil {
+		fmt.Printf("%s is up-to-date\n", dstPath)
+		return nil
+	}
+	fmt.Printf("%s is stale\n", dstPath)
+	fmt.Printf("downloading from %s\n", url)
+
+	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("download error: code %d, err %v", resp.StatusCode, err)
+	}
+	defer resp.Body.Close()
+
+	fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	dst := bufio.NewWriter(io.MultiWriter(fd, &dots{length: resp.ContentLength}))
+	_, copyErr := io.Copy(dst, resp.Body)
+	flushErr := dst.Flush()
+	fd.Close()
+	if copyErr != nil {
+		return copyErr
+	} else if flushErr != nil {
+		return flushErr
+	}
+
+	return db.Verify(dstPath)
+}
+
+type dots struct {
+	c       int64
+	length  int64
+	lastpct int64
+}
+
+func (d *dots) Write(buf []byte) (int, error) {
+	d.c += int64(len(buf))
+	pct := d.c * 10 / d.length * 10
+	if pct != d.lastpct {
+		if d.lastpct != 0 {
+			fmt.Print("...")
+		}
+		fmt.Print(pct, "%")
+		d.lastpct = pct
+	}
+	if pct == 100 {
+		fmt.Println()
+	}
+	return len(buf), nil
+}

internal/build/gosrc.go

@@ -1,81 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package build
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// EnsureGoSources ensures that path contains a file with the given SHA256 hash,
-// and if not, it downloads a fresh Go source package from upstream and replaces
-// path with it (if the hash matches).
-func EnsureGoSources(version string, hash []byte, path string) error {
-	// Sanity check the destination path to ensure we don't do weird things
-	if !strings.HasSuffix(path, ".tar.gz") {
-		return fmt.Errorf("destination path (%s) must end with .tar.gz", path)
-	}
-	// If the file exists, validate it's hash
-	if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all
-		hasher := sha256.New()
-		hasher.Write(archive)
-		have := hasher.Sum(nil)
-
-		if bytes.Equal(have, hash) {
-			fmt.Printf("Go %s [%x] available at %s\n", version, hash, path)
-			return nil
-		}
-		fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path)
-		if err := os.Remove(path); err != nil {
-			return err
-		}
-	}
-	// Archive missing or bad hash, download a new one
-	fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path)
-
-	res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version))
-	if err != nil || res.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err)
-	}
-	defer res.Body.Close()
-
-	archive, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return err
-	}
-	// Sanity check the downloaded archive, save if checks out
-	hasher := sha256.New()
-	hasher.Write(archive)
-
-	if have := hasher.Sum(nil); !bytes.Equal(have, hash) {
-		return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash)
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(path, archive, 0644); err != nil {
-		return err
-	}
-	fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path)
-	return nil
-}