build: run check_generate, lint, test for all modules

build/ci.go

@@ -64,6 +64,11 @@ import (
 )
 
 var (
+	goModules = []string{
+		".",
+		"./cmd/keeper",
+	}
+
 	// Files that end up in the geth*.zip archive.
 	gethArchiveFiles = []string{
 		"COPYING",
@@ -295,6 +300,7 @@ func doTest(cmdline []string) {
 	if *dlgo {
 		tc.Root = build.DownloadGo(csdb)
 	}
+
 	gotest := tc.Go("test")
 
 	// CI needs a bit more time for the statetests (default 45m).
@@ -323,11 +329,19 @@ func doTest(cmdline []string) {
 	}
 
 	packages := flag.CommandLine.Args()
-	if len(packages) == 0 {
+	if len(packages) > 0 {
-		packages = workspacePackagePatterns()
+		gotest.Args = append(gotest.Args, packages...)
+		build.MustRun(gotest)
+		return
+	}
+
+	// No packages specified, run all tests for all modules.
+	gotest.Args = append(gotest.Args, ".")
+	for _, mod := range goModules {
+		test := *gotest
+		test.Dir = mod
+		build.MustRun(&test)
 	}
-	gotest.Args = append(gotest.Args, packages...)
-	build.MustRun(gotest)
 }
 
 // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
@@ -364,27 +378,35 @@ func doCheckGenerate() {
 		protocPath      = downloadProtoc(*cachedir)
 		protocGenGoPath = downloadProtocGenGo(*cachedir)
 	)
-	c := tc.Go("generate", workspacePackagePatterns()...)
 	pathList := []string{filepath.Join(protocPath, "bin"), protocGenGoPath, os.Getenv("PATH")}
-	c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
-	build.MustRun(c)
 
-	// Check if generate file hashes have changed
+	for _, mod := range goModules {
-	generated, err := build.HashFolder(".", []string{"tests/testdata", "build/cache", ".git"})
+		c := tc.Go("generate", "./...")
-	if err != nil {
+		c.Env = append(c.Env, "PATH="+strings.Join(pathList, string(os.PathListSeparator)))
-		log.Fatalf("Error re-computing hashes: %v", err)
+		c.Dir = mod
-	}
+		build.MustRun(c)
-	updates := build.DiffHashes(hashes, generated)
+		// Check if generate file hashes have changed
-	for _, file := range updates {
+		generated, err := build.HashFolder(mod, []string{"tests/testdata", "build/cache", ".git"})
-		log.Printf("File changed: %s", file)
+		if err != nil {
-	}
+			log.Fatalf("Error re-computing hashes: %v", err)
-	if len(updates) != 0 {
+		}
-		log.Fatal("One or more generated files were updated by running 'go generate ./...'")
+		updates := build.DiffHashes(hashes, generated)
+		for _, file := range updates {
+			log.Printf("File changed: %s", file)
+		}
+		if len(updates) != 0 {
+			log.Fatal("One or more generated files were updated by running 'go generate ./...'")
+		}
+
 	}
 	fmt.Println("No stale files detected.")
 
 	// Run go mod tidy check.
-	build.MustRun(tc.Go("mod", "tidy", "-diff"))
+	for _, mod := range goModules {
+		tidy := tc.Go("mod", "tidy", "-diff")
+		tidy.Dir = mod
+		build.MustRun(tidy)
+	}
 	fmt.Println("No untidy module files detected.")
 }
 
@@ -425,20 +447,29 @@ func doLint(cmdline []string) {
 	)
 	flag.CommandLine.Parse(cmdline)
 
-	packages := flag.CommandLine.Args()
+	linter := downloadLinter(*cachedir)
-	if len(packages) == 0 {
+	linter, err := filepath.Abs(linter)
-		// Get module directories in workspace.
+	if err != nil {
-		packages = []string{"./..."}
+		log.Fatal(err)
-		modules := workspaceModules()
+	}
-		for _, m := range modules[1:] {
+	config, err := filepath.Abs(".golangci.yml")
-			dir := strings.TrimPrefix(m, modules[0])
+	if err != nil {
-			packages = append(packages, "."+dir+"/...")
+		log.Fatal(err)
-		}
 	}
 
-	linter := downloadLinter(*cachedir)
+	lflags := []string{"run", "--config", config}
-	lflags := []string{"run", "--config", ".golangci.yml"}
+	packages := flag.CommandLine.Args()
-	build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
+	if len(packages) > 0 {
+		build.MustRunCommandWithOutput(linter, append(lflags, packages...)...)
+	} else {
+		// Run for all modules in workspace.
+		for _, mod := range goModules {
+			args := append(lflags, "./...")
+			lintcmd := exec.Command(linter, args...)
+			lintcmd.Dir = mod
+			build.MustRunWithOutput(lintcmd)
+		}
+	}
 	fmt.Println("You have achieved perfection.")
 }
 
@@ -1176,31 +1207,3 @@ func doSanityCheck() {
 	csdb := download.MustLoadChecksums("build/checksums.txt")
 	csdb.DownloadAndVerifyAll()
 }
-
-// workspaceModules lists the module paths in the current work.
-func workspaceModules() []string {
-	listing, err := new(build.GoToolchain).Go("list", "-m").Output()
-	if err != nil {
-		log.Fatalf("go list failed:", err)
-	}
-	var modules []string
-	for _, m := range bytes.Split(listing, []byte("\n")) {
-		m = bytes.TrimSpace(m)
-		if len(m) > 0 {
-			modules = append(modules, string(m))
-		}
-	}
-	if len(modules) == 0 {
-		panic("no modules found")
-	}
-	return modules
-}
-
-func workspacePackagePatterns() []string {
-	modules := workspaceModules()
-	patterns := make([]string, len(modules))
-	for i, m := range modules {
-		patterns[i] = m + "/..."
-	}
-	return patterns
-}

internal/build/util.go

@@ -39,6 +39,9 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands")
 // MustRun executes the given command and exits the host process for
 // any error.
 func MustRun(cmd *exec.Cmd) {
+	if cmd.Dir != "" && cmd.Dir != "." {
+		fmt.Printf("(in %s) ", cmd.Dir)
+	}
 	fmt.Println(">>>", printArgs(cmd.Args))
 	if !*DryRunFlag {
 		cmd.Stderr = os.Stderr
@@ -71,6 +74,13 @@ func MustRunCommand(cmd string, args ...string) {
 // printed while it runs. This is useful for CI builds where the process will be stopped
 // when there is no output.
 func MustRunCommandWithOutput(cmd string, args ...string) {
+	MustRunWithOutput(exec.Command(cmd, args...))
+}
+
+// MustRunWithOutput runs the given command, and ensures that some output will be printed
+// while it runs. This is useful for CI builds where the process will be stopped when
+// there is no output.
+func MustRunWithOutput(cmd *exec.Cmd) {
 	interval := time.NewTicker(time.Minute)
 	done := make(chan struct{})
 	defer interval.Stop()
@@ -85,7 +95,7 @@ func MustRunCommandWithOutput(cmd string, args ...string) {
 			}
 		}
 	}()
-	MustRun(exec.Command(cmd, args...))
+	MustRun(cmd)
 }
 
 var warnedAboutGit bool