diff --git a/Makefile b/Makefile index 6b3cc8a0e5..4492228ac8 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,6 @@ -# This Makefile is meant to be used by people that do not usually work -# with Go source code. If you know what GOPATH is then you probably -# don't need to bother with make. - -.PHONY: tomo android ios tomo-cross swarm evm all test clean +.PHONY: tomo tomo-cross evm all test clean .PHONY: tomo-linux tomo-linux-386 tomo-linux-amd64 tomo-linux-mips64 tomo-linux-mips64le -.PHONY: tomo-linux-arm tomo-linux-arm-5 tomo-linux-arm-6 tomo-linux-arm-7 tomo-linux-arm64 .PHONY: tomo-darwin tomo-darwin-386 tomo-darwin-amd64 -.PHONY: tomo-windows tomo-windows-386 tomo-windows-amd64 GOBIN = $(shell pwd)/build/bin GOFMT = gofmt @@ -26,50 +20,27 @@ bootnode: @echo "Done building." @echo "Run \"$(GOBIN)/bootnode\" to launch a bootnode." -swarm: - build/env.sh go run build/ci.go install ./cmd/swarm +puppeth: + build/env.sh go run build/ci.go install ./cmd/puppeth @echo "Done building." - @echo "Run \"$(GOBIN)/swarm\" to launch swarm." + @echo "Run \"$(GOBIN)/puppeth\" to launch puppeth." all: build/env.sh go run build/ci.go install -android: - build/env.sh go run build/ci.go aar --local - @echo "Done building." - @echo "Import \"$(GOBIN)/tomo.aar\" to use the library." - -ios: - build/env.sh go run build/ci.go xcode --local - @echo "Done building." - @echo "Import \"$(GOBIN)/tomo.framework\" to use the library." - test: all build/env.sh go run build/ci.go test clean: rm -fr build/_workspace/pkg/ $(GOBIN)/* -# The devtools target installs tools required for 'go generate'. -# You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'. - -devtools: - env GOBIN= go get -u golang.org/x/tools/cmd/stringer - env GOBIN= go get -u github.com/kevinburke/go-bindata/go-bindata - env GOBIN= go get -u github.com/fjl/gencodec - env GOBIN= go get -u github.com/golang/protobuf/protoc-gen-go - env GOBIN= go install ./cmd/abigen - @type "npm" 2> /dev/null || echo 'Please install node.js and npm' - @type "solc" 2> /dev/null || echo 'Please install solc' - @type "protoc" 2> /dev/null || echo 'Please install protoc' - # Cross Compilation Targets (xgo) -tomo-cross: tomo-linux tomo-darwin tomo-windows tomo-android tomo-ios +tomo-cross: tomo-linux tomo-darwin @echo "Full cross compilation done:" @ls -ld $(GOBIN)/tomo-* -tomo-linux: tomo-linux-386 tomo-linux-amd64 tomo-linux-arm tomo-linux-mips64 tomo-linux-mips64le +tomo-linux: tomo-linux-386 tomo-linux-amd64 tomo-linux-mips64 tomo-linux-mips64le @echo "Linux cross compilation done:" @ls -ld $(GOBIN)/tomo-linux-* @@ -83,30 +54,6 @@ tomo-linux-amd64: @echo "Linux amd64 cross compilation done:" @ls -ld $(GOBIN)/tomo-linux-* | grep amd64 -tomo-linux-arm: tomo-linux-arm-5 tomo-linux-arm-6 tomo-linux-arm-7 tomo-linux-arm64 - @echo "Linux ARM cross compilation done:" - @ls -ld $(GOBIN)/tomo-linux-* | grep arm - -tomo-linux-arm-5: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-5 -v ./cmd/tomo - @echo "Linux ARMv5 cross compilation done:" - @ls -ld $(GOBIN)/tomo-linux-* | grep arm-5 - -tomo-linux-arm-6: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-6 -v ./cmd/tomo - @echo "Linux ARMv6 cross compilation done:" - @ls -ld $(GOBIN)/tomo-linux-* | grep arm-6 - -tomo-linux-arm-7: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-7 -v ./cmd/tomo - @echo "Linux ARMv7 cross compilation done:" - @ls -ld $(GOBIN)/tomo-linux-* | grep arm-7 - -tomo-linux-arm64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm64 -v ./cmd/tomo - @echo "Linux ARM64 cross compilation done:" - @ls -ld $(GOBIN)/tomo-linux-* | grep arm64 - tomo-linux-mips: build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips --ldflags '-extldflags "-static"' -v ./cmd/tomo @echo "Linux MIPS cross compilation done:" @@ -141,20 +88,6 @@ tomo-darwin-amd64: @echo "Darwin amd64 cross compilation done:" @ls -ld $(GOBIN)/tomo-darwin-* | grep amd64 -tomo-windows: tomo-windows-386 tomo-windows-amd64 - @echo "Windows cross compilation done:" - @ls -ld $(GOBIN)/tomo-windows-* - -tomo-windows-386: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/386 -v ./cmd/tomo - @echo "Windows 386 cross compilation done:" - @ls -ld $(GOBIN)/tomo-windows-* | grep 386 - -tomo-windows-amd64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/amd64 -v ./cmd/tomo - @echo "Windows amd64 cross compilation done:" - @ls -ld $(GOBIN)/geth-windows-* | grep amd64 - gofmt: $(GOFMT) -s -w $(GO_FILES) $(GIT) checkout vendor diff --git a/build/ci.go b/build/ci.go index 1d95ca0bf0..487f386896 100644 --- a/build/ci.go +++ b/build/ci.go @@ -26,14 +26,8 @@ Available commands are: install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables test [ -coverage ] [ packages... ] -- runs the tests lint -- runs certain pre-selected linters - archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts importkeys -- imports signing keys from env - debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package - nsis -- creates a Windows NSIS installer - aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive - xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework xgo [ -alltools ] [ options ] -- cross builds according to options - purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). @@ -41,9 +35,6 @@ For all commands, -n prevents execution of external programs (dry run mode). package main import ( - "bufio" - "bytes" - "encoding/base64" "flag" "fmt" "go/parser" @@ -53,21 +44,13 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "runtime" "strings" - "time" "github.com/ethereum/go-ethereum/internal/build" ) var ( - // Files that end up in the geth*.zip archive. - gethArchiveFiles = []string{ - "COPYING", - executablePath("geth"), - } - // Files that end up in the geth-alltools*.zip archive. allToolsArchiveFiles = []string{ "COPYING", @@ -80,49 +63,6 @@ var ( executablePath("swarm"), executablePath("wnode"), } - - // A debian package is created for all executables listed here. - debExecutables = []debExecutable{ - { - Name: "abigen", - Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", - }, - { - Name: "bootnode", - Description: "Ethereum bootnode.", - }, - { - Name: "evm", - Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", - }, - { - Name: "geth", - Description: "Ethereum CLI client.", - }, - { - Name: "puppeth", - Description: "Ethereum private network manager.", - }, - { - Name: "rlpdump", - Description: "Developer utility tool that prints RLP structures.", - }, - { - Name: "swarm", - Description: "Ethereum Swarm daemon and tools", - }, - { - Name: "wnode", - Description: "Ethereum Whisper diagnostic tool", - }, - } - - // Distros for which packages are created. - // Note: vivid is unsupported because there is no golang-1.6 package for it. - // Note: wily is unsupported because it was officially deprecated on lanchpad. - // Note: yakkety is unsupported because it was officially deprecated on lanchpad. - // Note: zesty is unsupported because it was officially deprecated on lanchpad. - debDistros = []string{"trusty", "xenial", "artful", "bionic"} ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -150,20 +90,8 @@ func main() { doTest(os.Args[2:]) case "lint": doLint(os.Args[2:]) - case "archive": - doArchive(os.Args[2:]) - case "debsrc": - doDebianSource(os.Args[2:]) - case "nsis": - doWindowsInstaller(os.Args[2:]) - case "aar": - doAndroidArchive(os.Args[2:]) - case "xcode": - doXCodeFramework(os.Args[2:]) case "xgo": doXgo(os.Args[2:]) - case "purge": - doPurge(os.Args[2:]) default: log.Fatal("unknown command ", os.Args[1]) } @@ -345,586 +273,6 @@ func doLint(cmdline []string) { } } -// Release Packaging - -func doArchive(cmdline []string) { - var ( - arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") - atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") - signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) - ext string - ) - flag.CommandLine.Parse(cmdline) - switch *atype { - case "zip": - ext = ".zip" - case "tar": - ext = ".tar.gz" - default: - log.Fatal("unknown archive type: ", atype) - } - - var ( - env = build.Env() - base = archiveBasename(*arch, env) - geth = "geth-" + base + ext - alltools = "geth-alltools-" + base + ext - ) - maybeSkipArchive(env) - if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { - log.Fatal(err) - } - if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { - log.Fatal(err) - } - for _, archive := range []string{geth, alltools} { - if err := archiveUpload(archive, *upload, *signer); err != nil { - log.Fatal(err) - } - } -} - -func archiveBasename(arch string, env build.Environment) string { - platform := runtime.GOOS + "-" + arch - if arch == "arm" { - platform += os.Getenv("GOARM") - } - if arch == "android" { - platform = "android-all" - } - if arch == "ios" { - platform = "ios-all" - } - return platform + "-" + archiveVersion(env) -} - -func archiveVersion(env build.Environment) string { - version := build.VERSION() - if isUnstableBuild(env) { - version += "-unstable" - } - if env.Commit != "" { - version += "-" + env.Commit[:8] - } - return version -} - -func archiveUpload(archive string, blobstore string, signer string) error { - // If signing was requested, generate the signature files - if signer != "" { - pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer)) - if err != nil { - return fmt.Errorf("invalid base64 %s", signer) - } - if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil { - return err - } - } - // If uploading to Azure was requested, push the archive possibly with its signature - if blobstore != "" { - auth := build.AzureBlobstoreConfig{ - Account: strings.Split(blobstore, "/")[0], - Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), - Container: strings.SplitN(blobstore, "/", 2)[1], - } - if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { - return err - } - if signer != "" { - if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { - return err - } - } - } - return nil -} - -// skips archiving for some build configurations. -func maybeSkipArchive(env build.Environment) { - if env.IsPullRequest { - log.Printf("skipping because this is a PR build") - os.Exit(0) - } - if env.IsCronJob { - log.Printf("skipping because this is a cron job") - os.Exit(0) - } - if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) - os.Exit(0) - } -} - -// Debian Packaging - -func doDebianSource(cmdline []string) { - var ( - signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`) - workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) - now = time.Now() - ) - flag.CommandLine.Parse(cmdline) - *workdir = makeWorkdir(*workdir) - env := build.Env() - maybeSkipArchive(env) - - // Import the signing key. - if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatal("invalid base64 PPA_SIGNING_KEY") - } - gpg := exec.Command("gpg", "--import") - gpg.Stdin = bytes.NewReader(key) - build.MustRun(gpg) - } - - // Create the packages. - for _, distro := range debDistros { - meta := newDebMetadata(distro, *signer, env, now) - pkgdir := stageDebianSource(*workdir, meta) - debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc") - debuild.Dir = pkgdir - build.MustRun(debuild) - - changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString()) - changes = filepath.Join(*workdir, changes) - if *signer != "" { - build.MustRunCommand("debsign", changes) - } - if *upload != "" { - build.MustRunCommand("dput", *upload, changes) - } - } -} - -func makeWorkdir(wdflag string) string { - var err error - if wdflag != "" { - err = os.MkdirAll(wdflag, 0744) - } else { - wdflag, err = ioutil.TempDir("", "geth-build-") - } - if err != nil { - log.Fatal(err) - } - return wdflag -} - -func isUnstableBuild(env build.Environment) bool { - if env.Tag != "" { - return false - } - return true -} - -type debMetadata struct { - Env build.Environment - - // go-ethereum version being built. Note that this - // is not the debian package version. The package version - // is constructed by VersionString. - Version string - - Author string // "name ", also selects signing key - Distro, Time string - Executables []debExecutable -} - -type debExecutable struct { - Name, Description string -} - -func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata { - if author == "" { - // No signing key, use default author. - author = "Ethereum Builds " - } - return debMetadata{ - Env: env, - Author: author, - Distro: distro, - Version: build.VERSION(), - Time: t.Format(time.RFC1123Z), - Executables: debExecutables, - } -} - -// Name returns the name of the metapackage that depends -// on all executable packages. -func (meta debMetadata) Name() string { - if isUnstableBuild(meta.Env) { - return "ethereum-unstable" - } - return "ethereum" -} - -// VersionString returns the debian version of the packages. -func (meta debMetadata) VersionString() string { - vsn := meta.Version - if meta.Env.Buildnum != "" { - vsn += "+build" + meta.Env.Buildnum - } - if meta.Distro != "" { - vsn += "+" + meta.Distro - } - return vsn -} - -// ExeList returns the list of all executable packages. -func (meta debMetadata) ExeList() string { - names := make([]string, len(meta.Executables)) - for i, e := range meta.Executables { - names[i] = meta.ExeName(e) - } - return strings.Join(names, ", ") -} - -// ExeName returns the package name of an executable package. -func (meta debMetadata) ExeName(exe debExecutable) string { - if isUnstableBuild(meta.Env) { - return exe.Name + "-unstable" - } - return exe.Name -} - -// ExeConflicts returns the content of the Conflicts field -// for executable packages. -func (meta debMetadata) ExeConflicts(exe debExecutable) string { - if isUnstableBuild(meta.Env) { - // Set up the conflicts list so that the *-unstable packages - // cannot be installed alongside the regular version. - // - // https://www.debian.org/doc/debian-policy/ch-relationships.html - // is very explicit about Conflicts: and says that Breaks: should - // be preferred and the conflicting files should be handled via - // alternates. We might do this eventually but using a conflict is - // easier now. - return "ethereum, " + exe.Name - } - return "" -} - -func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) { - pkg := meta.Name() + "-" + meta.VersionString() - pkgdir = filepath.Join(tmpdir, pkg) - if err := os.Mkdir(pkgdir, 0755); err != nil { - log.Fatal(err) - } - - // Copy the source code. - build.MustRunCommand("git", "checkout-index", "-a", "--prefix", pkgdir+string(filepath.Separator)) - - // Put the debian build files in place. - debian := filepath.Join(pkgdir, "debian") - build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta) - build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta) - build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta) - build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta) - build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta) - build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta) - for _, exe := range meta.Executables { - install := filepath.Join(debian, meta.ExeName(exe)+".install") - docs := filepath.Join(debian, meta.ExeName(exe)+".docs") - build.Render("build/deb.install", install, 0644, exe) - build.Render("build/deb.docs", docs, 0644, exe) - } - - return pkgdir -} - -// Windows installer - -func doWindowsInstaller(cmdline []string) { - // Parse the flags and make skip installer generation on PRs - var ( - arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") - signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) - workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) - ) - flag.CommandLine.Parse(cmdline) - *workdir = makeWorkdir(*workdir) - env := build.Env() - maybeSkipArchive(env) - - // Aggregate binaries that are included in the installer - var ( - devTools []string - allTools []string - gethTool string - ) - for _, file := range allToolsArchiveFiles { - if file == "COPYING" { // license, copied later - continue - } - allTools = append(allTools, filepath.Base(file)) - if filepath.Base(file) == "geth.exe" { - gethTool = file - } else { - devTools = append(devTools, file) - } - } - - // Render NSIS scripts: Installer NSIS contains two installer sections, - // first section contains the geth binary, second section holds the dev tools. - templateData := map[string]interface{}{ - "License": "COPYING", - "Geth": gethTool, - "DevTools": devTools, - } - build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) - build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) - build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) - build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) - build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) - build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755) - build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755) - - // Build the installer. This assumes that all the needed files have been previously - // built (don't mix building and packaging to keep cross compilation complexity to a - // minimum). - version := strings.Split(build.VERSION(), ".") - if env.Commit != "" { - version[2] += "-" + env.Commit[:8] - } - installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, env) + ".exe") - build.MustRunCommand("makensis.exe", - "/DOUTPUTFILE="+installer, - "/DMAJORVERSION="+version[0], - "/DMINORVERSION="+version[1], - "/DBUILDVERSION="+version[2], - "/DARCH="+*arch, - filepath.Join(*workdir, "geth.nsi"), - ) - - // Sign and publish installer. - if err := archiveUpload(installer, *upload, *signer); err != nil { - log.Fatal(err) - } -} - -// Android archives - -func doAndroidArchive(cmdline []string) { - var ( - local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) - signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) - deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) - upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) - ) - flag.CommandLine.Parse(cmdline) - env := build.Env() - - // Sanity check that the SDK and NDK are installed and set - if os.Getenv("ANDROID_HOME") == "" { - log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") - } - if os.Getenv("ANDROID_NDK") == "" { - log.Fatal("Please ensure ANDROID_NDK points to your Android NDK") - } - // Build the Android archive and Maven resources - build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) - build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) - build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) - - if *local { - // If we're building locally, copy bundle to build dir and skip Maven - os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) - return - } - meta := newMavenMetadata(env) - build.Render("build/mvn.pom", meta.Package+".pom", 0755, meta) - - // Skip Maven deploy and Azure upload for PR builds - maybeSkipArchive(env) - - // Sign and upload the archive to Azure - archive := "geth-" + archiveBasename("android", env) + ".aar" - os.Rename("geth.aar", archive) - - if err := archiveUpload(archive, *upload, *signer); err != nil { - log.Fatal(err) - } - // Sign and upload all the artifacts to Maven Central - os.Rename(archive, meta.Package+".aar") - if *signer != "" && *deploy != "" { - // Import the signing key into the local GPG instance - if b64key := os.Getenv(*signer); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } - gpg := exec.Command("gpg", "--import") - gpg.Stdin = bytes.NewReader(key) - build.MustRun(gpg) - } - // Upload the artifacts to Sonatype and/or Maven Central - repo := *deploy + "/service/local/staging/deploy/maven2" - if meta.Develop { - repo = *deploy + "/content/repositories/snapshots" - } - build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", - "-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh", - "-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar") - } -} - -func gomobileTool(subcmd string, args ...string) *exec.Cmd { - cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) - cmd.Args = append(cmd.Args, args...) - cmd.Env = []string{ - "GOPATH=" + build.GOPATH(), - "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), - } - for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") { - continue - } - cmd.Env = append(cmd.Env, e) - } - return cmd -} - -type mavenMetadata struct { - Version string - Package string - Develop bool - Contributors []mavenContributor -} - -type mavenContributor struct { - Name string - Email string -} - -func newMavenMetadata(env build.Environment) mavenMetadata { - // Collect the list of authors from the repo root - contribs := []mavenContributor{} - if authors, err := os.Open("AUTHORS"); err == nil { - defer authors.Close() - - scanner := bufio.NewScanner(authors) - for scanner.Scan() { - // Skip any whitespace from the authors list - line := strings.TrimSpace(scanner.Text()) - if line == "" || line[0] == '#' { - continue - } - // Split the author and insert as a contributor - re := regexp.MustCompile("([^<]+) <(.+)>") - parts := re.FindStringSubmatch(line) - if len(parts) == 3 { - contribs = append(contribs, mavenContributor{Name: parts[1], Email: parts[2]}) - } - } - } - // Render the version and package strings - version := build.VERSION() - if isUnstableBuild(env) { - version += "-SNAPSHOT" - } - return mavenMetadata{ - Version: version, - Package: "geth-" + version, - Develop: isUnstableBuild(env), - Contributors: contribs, - } -} - -// XCode frameworks - -func doXCodeFramework(cmdline []string) { - var ( - local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) - signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) - deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) - ) - flag.CommandLine.Parse(cmdline) - env := build.Env() - - // Build the iOS XCode framework - build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) - build.MustRun(gomobileTool("init")) - bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile") - - if *local { - // If we're building locally, use the build folder and stop afterwards - bind.Dir, _ = filepath.Abs(GOBIN) - build.MustRun(bind) - return - } - archive := "geth-" + archiveBasename("ios", env) - if err := os.Mkdir(archive, os.ModePerm); err != nil { - log.Fatal(err) - } - bind.Dir, _ = filepath.Abs(archive) - build.MustRun(bind) - build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive) - - // Skip CocoaPods deploy and Azure upload for PR builds - maybeSkipArchive(env) - - // Sign and upload the framework to Azure - if err := archiveUpload(archive+".tar.gz", *upload, *signer); err != nil { - log.Fatal(err) - } - // Prepare and upload a PodSpec to CocoaPods - if *deploy != "" { - meta := newPodMetadata(env, archive) - build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) - build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose") - } -} - -type podMetadata struct { - Version string - Commit string - Archive string - Contributors []podContributor -} - -type podContributor struct { - Name string - Email string -} - -func newPodMetadata(env build.Environment, archive string) podMetadata { - // Collect the list of authors from the repo root - contribs := []podContributor{} - if authors, err := os.Open("AUTHORS"); err == nil { - defer authors.Close() - - scanner := bufio.NewScanner(authors) - for scanner.Scan() { - // Skip any whitespace from the authors list - line := strings.TrimSpace(scanner.Text()) - if line == "" || line[0] == '#' { - continue - } - // Split the author and insert as a contributor - re := regexp.MustCompile("([^<]+) <(.+)>") - parts := re.FindStringSubmatch(line) - if len(parts) == 3 { - contribs = append(contribs, podContributor{Name: parts[1], Email: parts[2]}) - } - } - } - version := build.VERSION() - if isUnstableBuild(env) { - version += "-unstable." + env.Buildnum - } - return podMetadata{ - Archive: archive, - Version: version, - Commit: env.Commit, - Contributors: contribs, - } -} - // Cross compilation func doXgo(cmdline []string) { @@ -976,62 +324,3 @@ func xgoTool(args []string) *exec.Cmd { } return cmd } - -// Binary distribution cleanups - -func doPurge(cmdline []string) { - var ( - store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) - limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`) - ) - flag.CommandLine.Parse(cmdline) - - if env := build.Env(); !env.IsCronJob { - log.Printf("skipping because not a cron job") - os.Exit(0) - } - // Create the azure authentication and list the current archives - auth := build.AzureBlobstoreConfig{ - Account: strings.Split(*store, "/")[0], - Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), - Container: strings.SplitN(*store, "/", 2)[1], - } - blobs, err := build.AzureBlobstoreList(auth) - if err != nil { - log.Fatal(err) - } - // Iterate over the blobs, collect and sort all unstable builds - for i := 0; i < len(blobs); i++ { - if !strings.Contains(blobs[i].Name, "unstable") { - blobs = append(blobs[:i], blobs[i+1:]...) - i-- - } - } - for i := 0; i < len(blobs); i++ { - for j := i + 1; j < len(blobs); j++ { - iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified) - if err != nil { - log.Fatal(err) - } - jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified) - if err != nil { - log.Fatal(err) - } - if iTime.After(jTime) { - blobs[i], blobs[j] = blobs[j], blobs[i] - } - } - } - // Filter out all archives more recent that the given threshold - for i, blob := range blobs { - timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified) - if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour { - blobs = blobs[:i] - break - } - } - // Delete all marked as such and return - if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { - log.Fatal(err) - } -} diff --git a/internal/build/archive.go b/internal/build/archive.go deleted file mode 100644 index ac680ba63d..0000000000 --- a/internal/build/archive.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "archive/tar" - "archive/zip" - "compress/gzip" - "fmt" - "io" - "os" - "path/filepath" - "strings" -) - -type Archive interface { - // Directory adds a new directory entry to the archive and sets the - // directory for subsequent calls to Header. - Directory(name string) error - - // Header adds a new file to the archive. The file is added to the directory - // set by Directory. The content of the file must be written to the returned - // writer. - Header(os.FileInfo) (io.Writer, error) - - // Close flushes the archive and closes the underlying file. - Close() error -} - -func NewArchive(file *os.File) (Archive, string) { - switch { - case strings.HasSuffix(file.Name(), ".zip"): - return NewZipArchive(file), strings.TrimSuffix(file.Name(), ".zip") - case strings.HasSuffix(file.Name(), ".tar.gz"): - return NewTarballArchive(file), strings.TrimSuffix(file.Name(), ".tar.gz") - default: - return nil, "" - } -} - -// AddFile appends an existing file to an archive. -func AddFile(a Archive, file string) error { - fd, err := os.Open(file) - if err != nil { - return err - } - defer fd.Close() - fi, err := fd.Stat() - if err != nil { - return err - } - w, err := a.Header(fi) - if err != nil { - return err - } - if _, err := io.Copy(w, fd); err != nil { - return err - } - return nil -} - -// WriteArchive creates an archive containing the given files. -func WriteArchive(name string, files []string) (err error) { - archfd, err := os.Create(name) - if err != nil { - return err - } - - defer func() { - archfd.Close() - // Remove the half-written archive on failure. - if err != nil { - os.Remove(name) - } - }() - archive, basename := NewArchive(archfd) - if archive == nil { - return fmt.Errorf("unknown archive extension") - } - fmt.Println(name) - if err := archive.Directory(basename); err != nil { - return err - } - for _, file := range files { - fmt.Println(" +", filepath.Base(file)) - if err := AddFile(archive, file); err != nil { - return err - } - } - return archive.Close() -} - -type ZipArchive struct { - dir string - zipw *zip.Writer - file io.Closer -} - -func NewZipArchive(w io.WriteCloser) Archive { - return &ZipArchive{"", zip.NewWriter(w), w} -} - -func (a *ZipArchive) Directory(name string) error { - a.dir = name + "/" - return nil -} - -func (a *ZipArchive) Header(fi os.FileInfo) (io.Writer, error) { - head, err := zip.FileInfoHeader(fi) - if err != nil { - return nil, fmt.Errorf("can't make zip header: %v", err) - } - head.Name = a.dir + head.Name - head.Method = zip.Deflate - w, err := a.zipw.CreateHeader(head) - if err != nil { - return nil, fmt.Errorf("can't add zip header: %v", err) - } - return w, nil -} - -func (a *ZipArchive) Close() error { - if err := a.zipw.Close(); err != nil { - return err - } - return a.file.Close() -} - -type TarballArchive struct { - dir string - tarw *tar.Writer - gzw *gzip.Writer - file io.Closer -} - -func NewTarballArchive(w io.WriteCloser) Archive { - gzw := gzip.NewWriter(w) - tarw := tar.NewWriter(gzw) - return &TarballArchive{"", tarw, gzw, w} -} - -func (a *TarballArchive) Directory(name string) error { - a.dir = name + "/" - return a.tarw.WriteHeader(&tar.Header{ - Name: a.dir, - Mode: 0755, - Typeflag: tar.TypeDir, - }) -} - -func (a *TarballArchive) Header(fi os.FileInfo) (io.Writer, error) { - head, err := tar.FileInfoHeader(fi, "") - if err != nil { - return nil, fmt.Errorf("can't make tar header: %v", err) - } - head.Name = a.dir + head.Name - if err := a.tarw.WriteHeader(head); err != nil { - return nil, fmt.Errorf("can't add tar header: %v", err) - } - return a.tarw, nil -} - -func (a *TarballArchive) Close() error { - if err := a.tarw.Close(); err != nil { - return err - } - if err := a.gzw.Close(); err != nil { - return err - } - return a.file.Close() -} diff --git a/internal/build/azure.go b/internal/build/azure.go deleted file mode 100644 index 2081a9a0b1..0000000000 --- a/internal/build/azure.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "fmt" - "os" - - storage "github.com/Azure/azure-storage-go" -) - -// AzureBlobstoreConfig is an authentication and configuration struct containing -// the data needed by the Azure SDK to interact with a speicifc container in the -// blobstore. -type AzureBlobstoreConfig struct { - Account string // Account name to authorize API requests with - Token string // Access token for the above account - Container string // Blob container to upload files into -} - -// AzureBlobstoreUpload uploads a local file to the Azure Blob Storage. Note, this -// method assumes a max file size of 64MB (Azure limitation). Larger files will -// need a multi API call approach implemented. -// -// See: https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx#Anchor_3 -func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) error { - if *DryRunFlag { - fmt.Printf("would upload %q to %s/%s/%s\n", path, config.Account, config.Container, name) - return nil - } - // Create an authenticated client against the Azure cloud - rawClient, err := storage.NewBasicClient(config.Account, config.Token) - if err != nil { - return err - } - client := rawClient.GetBlobService() - - // Stream the file to upload into the designated blobstore container - in, err := os.Open(path) - if err != nil { - return err - } - defer in.Close() - - info, err := in.Stat() - if err != nil { - return err - } - return client.CreateBlockBlobFromReader(config.Container, name, uint64(info.Size()), in, nil) -} - -// AzureBlobstoreList lists all the files contained within an azure blobstore. -func AzureBlobstoreList(config AzureBlobstoreConfig) ([]storage.Blob, error) { - // Create an authenticated client against the Azure cloud - rawClient, err := storage.NewBasicClient(config.Account, config.Token) - if err != nil { - return nil, err - } - client := rawClient.GetBlobService() - - // List all the blobs from the container and return them - container := client.GetContainerReference(config.Container) - - blobs, err := container.ListBlobs(storage.ListBlobsParameters{ - MaxResults: 1024 * 1024 * 1024, // Yes, fetch all of them - Timeout: 3600, // Yes, wait for all of them - }) - if err != nil { - return nil, err - } - return blobs.Blobs, nil -} - -// AzureBlobstoreDelete iterates over a list of files to delete and removes them -// from the blobstore. -func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []storage.Blob) error { - if *DryRunFlag { - for _, blob := range blobs { - fmt.Printf("would delete %s (%s) from %s/%s\n", blob.Name, blob.Properties.LastModified, config.Account, config.Container) - } - return nil - } - // Create an authenticated client against the Azure cloud - rawClient, err := storage.NewBasicClient(config.Account, config.Token) - if err != nil { - return err - } - client := rawClient.GetBlobService() - - // Iterate over the blobs and delete them - for _, blob := range blobs { - if err := client.DeleteBlob(config.Container, blob.Name, nil); err != nil { - return err - } - } - return nil -} diff --git a/internal/build/pgp.go b/internal/build/pgp.go deleted file mode 100644 index 79ab9c06f1..0000000000 --- a/internal/build/pgp.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// signFile reads the contents of an input file and signs it (in armored format) -// with the key provided, placing the signature into the output file. - -package build - -import ( - "bytes" - "fmt" - "os" - - "golang.org/x/crypto/openpgp" -) - -// PGPSignFile parses a PGP private key from the specified string and creates a -// signature file into the output parameter of the input file. -// -// Note, this method assumes a single key will be container in the pgpkey arg, -// furthermore that it is in armored format. -func PGPSignFile(input string, output string, pgpkey string) error { - // Parse the keyring and make sure we only have a single private key in it - keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey)) - if err != nil { - return err - } - if len(keys) != 1 { - return fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1) - } - // Create the input and output streams for signing - in, err := os.Open(input) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(output) - if err != nil { - return err - } - defer out.Close() - - // Generate the signature and return - return openpgp.ArmoredDetachSign(out, keys[0], in, nil) -} diff --git a/internal/build/util.go b/internal/build/util.go index c6e059f0df..a24099acfb 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,7 +29,6 @@ import ( "path/filepath" "runtime" "strings" - "text/template" ) var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") @@ -60,15 +59,6 @@ func GOPATH() string { return os.Getenv("GOPATH") } -// VERSION returns the content of the VERSION file. -func VERSION() string { - version, err := ioutil.ReadFile("VERSION") - if err != nil { - log.Fatal(err) - } - return string(bytes.TrimSpace(version)) -} - var warnedAboutGit bool // RunGit runs a git subcommand and returns its output. @@ -98,34 +88,6 @@ func readGitFile(file string) string { return strings.TrimSpace(string(content)) } -// Render renders the given template file into outputFile. -func Render(templateFile, outputFile string, outputPerm os.FileMode, x interface{}) { - tpl := template.Must(template.ParseFiles(templateFile)) - render(tpl, outputFile, outputPerm, x) -} - -// RenderString renders the given template string into outputFile. -func RenderString(templateContent, outputFile string, outputPerm os.FileMode, x interface{}) { - tpl := template.Must(template.New("").Parse(templateContent)) - render(tpl, outputFile, outputPerm, x) -} - -func render(tpl *template.Template, outputFile string, outputPerm os.FileMode, x interface{}) { - if err := os.MkdirAll(filepath.Dir(outputFile), 0755); err != nil { - log.Fatal(err) - } - out, err := os.OpenFile(outputFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, outputPerm) - if err != nil { - log.Fatal(err) - } - if err := tpl.Execute(out, x); err != nil { - log.Fatal(err) - } - if err := out.Close(); err != nil { - log.Fatal(err) - } -} - // CopyFile copies a file. func CopyFile(dst, src string, mode os.FileMode) { if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {