diff --git a/appveyor.yml b/appveyor.yml index 41c70491b4..f423e6c5b1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,6 +24,7 @@ for: - image: Ubuntu build_script: - go run build/ci.go lint + - go run build/ci.go generate - go run build/ci.go install -dlgo test_script: - go run build/ci.go test -dlgo -short diff --git a/build/checksums.txt b/build/checksums.txt index d099e53156..855ce5391d 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -99,3 +99,24 @@ d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767 go1.19.6.src.t # version:ppa-builder-2 1.21.9 # https://go.dev/dl/ 58f0c5ced45a0012bce2ff7a9df03e128abcc8818ebabe5027bb92bafe20e421 go1.21.9.src.tar.gz + +# version:protoc 27.1 +8809c2ec85368c6b6e9af161b6771a153aa92670a24adbe46dd34fa02a04df2f protoc-27.1-linux-aarch_64.zip +5d21979a6d27475e810b76b88863d1e784fa01ffb15e511a3ec5bd1924d89426 protoc-27.1-linux-ppcle_64.zip +84d8852750ed186dc4a057a1a86bcac409be5362d6af04770f42367fee6b7bc1 protoc-27.1-linux-s390_64.zip +2f028796ff5741691650e0eea290e61ff2f1c0d87f8d31fe45ef47fd967cef0c protoc-27.1-linux-x86_32.zip +8970e3d8bbd67d53768fe8c2e3971bdd71e51cfe2001ca06dacad17258a7dae3 protoc-27.1-linux-x86_64.zip +03b7af1bf469e7285dc51976ee5fa99412704dbd1c017105114852a37b165c12 protoc-27.1-osx-aarch_64.zip +f14d3973cf13283d07c520ed6f4c12405ad41b9efd18089a1c74897037d742b5 protoc-27.1-osx-universal_binary.zip +8520d944f3a3890fa296a3b3b0d4bb18337337e2526bbbf1b507eeea3c2a1ec4 protoc-27.1-osx-x86_64.zip +6263718ff96547b8392a079f6fdf02a4156f2e8d13cd51649a0d03fb7afa2de8 protoc-27.1-win32.zip +da531c51ccd1290d8d34821f0ce4e219c7fbaa6f9825f5a3fb092a9d03fe6206 protoc-27.1-win64.zip + +# version:protoc-gen-go 1.34.2 +9b48d8f90add02e8e94e14962fed74e7ce2b2d6bda4dd42f1f4fbccf0f766f1a protoc-gen-go.v1.34.2.darwin.amd64.tar.gz +17aca7f948dbb624049030cf841e35895cf34183ba006e721247fdeb95ff2780 protoc-gen-go.v1.34.2.darwin.arm64.tar.gz +a191849433fd489f1d44f37788d762658f3f5fb225f3a85d4ce6ba32666703ed protoc-gen-go.v1.34.2.linux.386.tar.gz +b87bc134dee55576a842141bf0ed27761c635d746780fce5dee038c6dd16554f protoc-gen-go.v1.34.2.linux.amd64.tar.gz +63d400167e75ab9f6690688f6fdc6a9455aa20bc1faa71e32149dbd322f7f198 protoc-gen-go.v1.34.2.linux.arm64.tar.gz +56e7675816db6e62be4f833a51544d5716b8420c462515579e05ca8444ab06ed protoc-gen-go.v1.34.2.windows.386.zip +abafd39612177dd4e9a65207cadd5374a9352d8611e8e040f8462fcfa3010daf protoc-gen-go.v1.34.2.windows.amd64.zip diff --git a/build/ci.go b/build/ci.go index 9a2532f51f..c29021c325 100644 --- a/build/ci.go +++ b/build/ci.go @@ -39,9 +39,11 @@ package main import ( "bytes" + "crypto/sha256" "encoding/base64" "flag" "fmt" + "io" "log" "os" "os/exec" @@ -169,6 +171,8 @@ func main() { doPurge(os.Args[2:]) case "sanitycheck": doSanityCheck() + case "generate": + doGenerate() default: log.Fatal("unknown command ", os.Args[1]) } @@ -345,6 +349,77 @@ func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { return filepath.Join(cachedir, base) } +// hashDir iterates all files under the top-level project directory +// computing the hash of each file (excluding files within the tests +// subrepo) +func hashDir() (map[string]common.Hash, error) { + res := make(map[string]common.Hash) + err := filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error { + if strings.HasPrefix(path, filepath.FromSlash("tests/testdata")) { + return nil + } + if d.IsDir() { + return nil + } + // open the file and hash it + f, err := os.OpenFile(path, os.O_RDONLY, 0666) + if err != nil { + return err + } + hasher := sha256.New() + if _, err := io.Copy(hasher, f); err != nil { + return err + } + res[path] = common.Hash(hasher.Sum(nil)) + return nil + }) + if err != nil { + return nil, err + } + return res, nil +} + +// doGenerate ensures that re-generating generated files does not cause +// any mutations in the source file tree: i.e. all generated files were +// updated and committed. Any stale generated files are updated. +func doGenerate() { + tc := new(build.GoToolchain) + cachedir := flag.String("cachedir", "./build/cache", "directory for caching protoc/protoc-gen-go binaries.") + + protocPath := downloadProtoc(*cachedir) + protocGenGoPath := downloadProtocGenGo(*cachedir) + + preHashes, err := hashDir() + if err != nil { + log.Fatal("failed to compute map of source hashes", "err", err) + } + + c := tc.Go("generate", "./...") + newPathEnv := os.Getenv("PATH") + string(os.PathListSeparator) + protocPath + string(os.PathSeparator) + "bin" + newPathEnv += string(os.PathListSeparator) + protocGenGoPath + c.Env = append(c.Env, "PATH="+newPathEnv) + build.MustRun(c) + + postHashes, err := hashDir() + if err != nil { + log.Fatal("error computing source tree file hashes", "err", err) + } + + updates := []string{} + for path, postHash := range postHashes { + preHash, ok := preHashes[path] + if !ok || preHash != postHash { + updates = append(updates, path) + } + } + for _, updatedFile := range updates { + fmt.Fprintf(os.Stderr, "updated generated file %s\n", updatedFile) + } + if len(updates) != 0 { + log.Fatal("One or more generated files were updated by running 'go generate ./...'") + } +} + // doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { var ( @@ -390,6 +465,96 @@ func downloadLinter(cachedir string) string { return filepath.Join(cachedir, base, "golangci-lint") } +// protocArchiveBaseName returns the name of the protoc archive file for +// the current system, stripped of version and file suffix. +func protocArchiveBaseName() (string, error) { + switch runtime.GOOS + "-" + runtime.GOARCH { + case "windows-amd64": + return "win64", nil + case "windows-386": + return "win32", nil + case "linux-arm64": + return "linux-aarch_64", nil + case "linux-386": + return "linux-x86_32", nil + case "linux-amd64": + return "linux-x86_64", nil + case "darwin-arm64": + return "osx-aarch_64", nil + case "darwin-amd64": + return "osx-x86_64", nil + default: + return "", fmt.Errorf("no prebuilt release of protoc available for this system (os: %s, arch: %s)", runtime.GOOS, runtime.GOARCH) + } +} + +// downloadProtocGenGo downloads protoc-gen-go, which is used by protoc +// in the generate command. It returns the full path of the directory +// containing the 'protoc-gen-go' executable. +func downloadProtocGenGo(cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + version, err := build.Version(csdb, "protoc-gen-go") + if err != nil { + log.Fatal(err) + } + baseName := fmt.Sprintf("protoc-gen-go.v%s.%s.%s", version, runtime.GOOS, runtime.GOARCH) + archiveName := baseName + if runtime.GOOS == "windows" { + archiveName += ".zip" + } else { + archiveName += ".tar.gz" + } + + url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName) + + archivePath := path.Join(cachedir, archiveName) + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + extractDest := filepath.Join(cachedir, baseName) + if err := build.ExtractArchive(archivePath, extractDest); err != nil { + log.Fatal(err) + } + extractDest, err = filepath.Abs(extractDest) + if err != nil { + log.Fatal("error resolving absolute path for protoc", "err", err) + } + return extractDest +} + +// downloadProtoc downloads the prebuilt protoc binary used to lint generated +// files as a CI step. It returns the full path to the directory containing +// the protoc executable. +func downloadProtoc(cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + version, err := build.Version(csdb, "protoc") + if err != nil { + log.Fatal(err) + } + baseName, err := protocArchiveBaseName() + if err != nil { + log.Fatal(err) + } + + fileName := fmt.Sprintf("protoc-%s-%s", version, baseName) + archiveFileName := fileName + ".zip" + url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName) + archivePath := filepath.Join(cachedir, archiveFileName) + + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + extractDest := filepath.Join(cachedir, fileName) + if err := build.ExtractArchive(archivePath, extractDest); err != nil { + log.Fatal(err) + } + extractDest, err = filepath.Abs(extractDest) + if err != nil { + log.Fatal("error resolving absolute path for protoc", "err", err) + } + return extractDest +} + // Release Packaging func doArchive(cmdline []string) { var ( diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index db058e847c..aa66dc49ff 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -199,7 +199,7 @@ type Hooks struct { // for tracing and reporting. type BalanceChangeReason byte -//go:generate stringer -type=BalanceChangeReason -output gen_balance_change_reason_stringer.go +//go:generate go run golang.org/x/tools/cmd/stringer -type=BalanceChangeReason -output gen_balance_change_reason_stringer.go const ( BalanceChangeUnspecified BalanceChangeReason = 0 diff --git a/internal/build/archive.go b/internal/build/archive.go index 5b37c0edfa..645921c690 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -272,17 +272,22 @@ func extractFile(arpath string, armode os.FileMode, data io.Reader, dest string) return fmt.Errorf("path %q escapes archive destination", target) } - // Ensure the destination directory exists. + // Remove the preivously-extracted file if it exists + if err := os.RemoveAll(target); err != nil { + return err + } + + // Recreate the destination directory if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err } // Copy file data. - file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, armode) + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY, armode) if err != nil { return err } - if _, err := io.Copy(file, data); err != nil { + if _, err = io.Copy(file, data); err != nil { file.Close() os.Remove(target) return err