diff --git a/Makefile b/Makefile index 3b748a0a0b..cf6493922c 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,10 @@ lint: ## Run linters. check_tidy: ## Run 'go mod tidy'. $(GORUN) build/ci.go check_tidy +#? check_generate: Verify everything is 'go generate'-ed +check_generate: ## Run 'go generate ./...'. + $(GORUN) build/ci.go check_generate + #? clean: Clean go cache, built executables, and the auto generated folder. clean: rm -fr build/_workspace/pkg/ $(GOBIN)/* diff --git a/build/ci.go b/build/ci.go index 3c0660a71a..9b13d4bff5 100644 --- a/build/ci.go +++ b/build/ci.go @@ -26,6 +26,7 @@ Available commands are: lint -- runs certain pre-selected linters check_tidy -- verifies that everything is 'go mod tidy'-ed + check_generate -- verifies that everything is 'go generate'-ed install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables test [ -coverage ] [ packages... ] -- runs the tests @@ -44,6 +45,7 @@ import ( "log" "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -93,6 +95,8 @@ func main() { doLint(os.Args[2:]) case "check_tidy": doCheckTidy() + case "check_generate": + doCheckGenerate() case "xgo": doXgo(os.Args[2:]) default: @@ -270,6 +274,45 @@ func doCheckTidy() { fmt.Println("No untidy module files detected.") } +// doCheckGenerate ensures that re-generating generated files does not cause +// any mutations in the source file tree. +func doCheckGenerate() { + var ( + cachedir = flag.String("cachedir", "./build/cache", "directory for caching binaries.") + ) + // Compute the origin hashes of all the files + var hashes map[string][32]byte + + var err error + hashes, err = build.HashFolder(".", []string{"tests/testdata", "build/cache"}) + if err != nil { + log.Fatal("Error computing hashes", "err", err) + } + // Run any go generate steps we might be missing + var ( + protocPath = downloadProtoc(*cachedir) + protocGenGoPath = downloadProtocGenGo(*cachedir) + ) + c := new(build.GoToolchain).Go("generate", "./...") + pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")} + c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator))) + build.MustRun(c) + + // Check if generate file hashes have changed + generated, err := build.HashFolder(".", []string{"tests/testdata", "build/cache"}) + if err != nil { + log.Fatalf("Error re-computing hashes: %v", err) + } + updates := build.DiffHashes(hashes, generated) + for _, file := range updates { + log.Printf("File changed: %s", file) + } + if len(updates) != 0 { + log.Fatal("One or more generated files were updated by running 'go generate ./...'") + } + fmt.Println("No stale files detected.") +} + // doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { var ( @@ -315,6 +358,96 @@ func downloadLinter(cachedir string) string { return filepath.Join(cachedir, base, "golangci-lint") } +// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc +// in the generate command. It returns the full path of the directory +// containing the 'protoc-gen-go' executable. +func downloadProtocGenGo(cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + version, err := build.Version(csdb, "protoc-gen-go") + if err != nil { + log.Fatal(err) + } + baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH) + archiveName := baseName + if runtime.GOOS == "windows" { + archiveName += ".zip" + } else { + archiveName += ".tar.gz" + } + + url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName) + + archivePath := path.Join(cachedir, archiveName) + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + extractDest := filepath.Join(cachedir, baseName) + if err := build.ExtractArchive(archivePath, extractDest); err != nil { + log.Fatal(err) + } + extractDest, err = filepath.Abs(extractDest) + if err != nil { + log.Fatal("error resolving absolute path for protoc", "err", err) + } + return extractDest +} + +// protocArchiveBaseName returns the name of the protoc archive file for +// the current system, stripped of version and file suffix. +func protocArchiveBaseName() (string, error) { + switch runtime.GOOS + "-" + runtime.GOARCH { + case "windows-amd64": + return "win64", nil + case "windows-386": + return "win32", nil + case "linux-arm64": + return "linux-aarch_64", nil + case "linux-386": + return "linux-x86_32", nil + case "linux-amd64": + return "linux-x86_64", nil + case "darwin-arm64": + return "osx-aarch_64", nil + case "darwin-amd64": + return "osx-x86_64", nil + default: + return "", fmt.Errorf("no prebuilt release of protoc available for this system (os: %s, arch: %s)", runtime.GOOS, runtime.GOARCH) + } +} + +// downloadProtoc downloads the prebuilt protoc binary used to lint generated +// files as a CI step. It returns the full path to the directory containing +// the protoc executable. +func downloadProtoc(cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + version, err := build.Version(csdb, "protoc") + if err != nil { + log.Fatal(err) + } + baseName, err := protocArchiveBaseName() + if err != nil { + log.Fatal(err) + } + + fileName := fmt.Sprintf("protoc-%s-%s", version, baseName) + archiveFileName := fileName + ".zip" + url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName) + archivePath := filepath.Join(cachedir, archiveFileName) + + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + extractDest := filepath.Join(cachedir, fileName) + if err := build.ExtractArchive(archivePath, extractDest); err != nil { + log.Fatal(err) + } + extractDest, err = filepath.Abs(extractDest) + if err != nil { + log.Fatal("error resolving absolute path for protoc", "err", err) + } + return extractDest +} + // Cross compilation func doXgo(cmdline []string) { var ( diff --git a/build/tools/tools.go b/build/tools/tools.go new file mode 100644 index 0000000000..e9e2241d2f --- /dev/null +++ b/build/tools/tools.go @@ -0,0 +1,27 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build tools +// +build tools + +package tools + +import ( + // Tool imports for go:generate. + _ "github.com/fjl/gencodec" + _ "golang.org/x/tools/cmd/stringer" + _ "google.golang.org/protobuf/cmd/protoc-gen-go" +) diff --git a/go.mod b/go.mod index 9b6faf749c..a4a458bbeb 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 github.com/ethereum/c-kzg-4844 v0.4.0 + github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/go-yaml/yaml v2.1.0+incompatible github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c @@ -57,6 +58,7 @@ require ( github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/urfave/cli/v2 v2.27.5 + google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -68,6 +70,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect + github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect github.com/go-ole/go-ole v1.2.5 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/uuid v1.3.0 // indirect @@ -91,7 +94,6 @@ require ( golang.org/x/net v0.25.0 // indirect golang.org/x/term v0.26.0 // indirect golang.org/x/text v0.20.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index baa6733abe..8e3cc32731 100644 --- a/go.sum +++ b/go.sum @@ -47,9 +47,13 @@ github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= +github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= +github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=