forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-28 07:36:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge tag v1.9.24 into geth-bor

Browse source
This commit is contained in:
Jaynti Kanani 2020-11-20 10:52:42 +05:30
commit b395595c8b
No known key found for this signature in database
GPG key ID: 4396982C976BAE5E
97 changed files with 4805 additions and 1034 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.github/CODEOWNERS vendored
View file

@ -3,21 +3,20 @@
accounts/usbwallet @karalabe
accounts/scwallet @gballet
accounts/abi @gballet
accounts/abi @gballet @MariusVanDerWijden
cmd/clef @holiman
cmd/puppeth @karalabe
consensus @karalabe
core/ @karalabe @holiman @rjl493456442
dashboard/ @kurkomisi
eth/ @karalabe @holiman @rjl493456442
graphql/ @gballet
les/ @zsfelfoldi @rjl493456442
light/ @zsfelfoldi @rjl493456442
mobile/ @karalabe @ligi
node/ @fjl @renaynay
p2p/ @fjl @zsfelfoldi
rpc/ @fjl @holiman
p2p/simulations @zelig @janos @justelad
p2p/protocols @zelig @janos @justelad
p2p/testing @zelig @janos @justelad
p2p/simulations @fjl
p2p/protocols @fjl
p2p/testing @fjl
signer/ @holiman
whisper/ @gballet @gluk256

112
.travis.yml
View file

@ -16,7 +16,7 @@ jobs:
- stage: lint
os: linux
dist: xenial
go: 1.14.x
go: 1.15.x
env:
- lint
git:
@ -24,65 +24,12 @@ jobs:
script:
- go run build/ci.go lint
- stage: build
os: linux
dist: xenial
go: 1.13.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
# These are the latest Go versions.
- stage: build
os: linux
arch: amd64
dist: xenial
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
if: type = pull_request
os: linux
arch: arm64
dist: xenial
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
os: osx
osx_image: xcode11.3
go: 1.14.x
env:
- GO111MODULE=on
script:
- echo "Increase the maximum number of open file descriptors on macOS"
- NOFILE=20480
- sudo sysctl -w kern.maxfiles=$NOFILE
- sudo sysctl -w kern.maxfilesperproc=$NOFILE
- sudo launchctl limit maxfiles $NOFILE $NOFILE
- sudo launchctl limit maxfiles
- ulimit -S -n $NOFILE
- ulimit -n
- unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
- go run build/ci.go install
- go run build/ci.go test -coverage $TEST_PACKAGES
# This builder does the Ubuntu PPA upload
- stage: build
if: type = push
os: linux
dist: xenial
go: 1.14.x
go: 1.15.x
env:
- ubuntu-ppa
- GO111MODULE=on
@ -107,7 +54,7 @@ jobs:
os: linux
dist: xenial
sudo: required
go: 1.14.x
go: 1.15.x
env:
- azure-linux
- GO111MODULE=on
@ -119,22 +66,22 @@ jobs:
- gcc-multilib
script:
# Build for the primary platforms that Trusty can manage
- go run build/ci.go install
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
- go run build/ci.go install -arch 386
- go run build/ci.go install -dlgo -arch 386
- go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# Switch over GCC to cross compilation (breaks 386, hence why do it here only)
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- sudo ln -s /usr/include/asm-generic /usr/include/asm
- GOARM=5 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=5 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
- GOARM=6 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=6 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
- GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
- GOARM=7 go run build/ci.go install -arch arm -cc arm-linux-gnueabihf-gcc
- GOARM=7 go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabihf-gcc
- GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
- go run build/ci.go install -arch arm64 -cc aarch64-linux-gnu-gcc
- go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
# This builder does the Linux Azure MIPS xgo uploads
@ -144,7 +91,7 @@ jobs:
dist: xenial
services:
- docker
go: 1.14.x
go: 1.15.x
env:
- azure-linux-mips
- GO111MODULE=on
@ -192,7 +139,7 @@ jobs:
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- curl https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz | tar -xz
- curl https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz | tar -xz
- export PATH=`pwd`/go/bin:$PATH
- export GOROOT=`pwd`/go
- export GOPATH=$HOME/go
@ -210,7 +157,7 @@ jobs:
- stage: build
if: type = push
os: osx
go: 1.14.x
go: 1.15.x
env:
- azure-osx
- azure-ios
@ -219,7 +166,7 @@ jobs:
git:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go install
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
# Build the iOS framework and upload it to CocoaPods and Azure
@ -237,12 +184,43 @@ jobs:
- export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
- go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
# These builders run the tests
- stage: build
os: linux
arch: amd64
dist: xenial
go: 1.15.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
if: type = pull_request
os: linux
arch: arm64
dist: xenial
go: 1.15.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
- stage: build
os: linux
dist: xenial
go: 1.14.x
env:
- GO111MODULE=on
script:
- go run build/ci.go test -coverage $TEST_PACKAGES
# This builder does the Azure archive purges to avoid accumulating junk
- stage: build
if: type = cron
os: linux
dist: xenial
go: 1.14.x
go: 1.15.x
env:
- azure-purge
- GO111MODULE=on

5
accounts/abi/bind/base.go
View file

@ -152,7 +152,10 @@ func (c *BoundContract) Call(opts *CallOpts, results *[]interface{}, method stri
}
} else {
output, err = c.caller.CallContract(ctx, msg, opts.BlockNumber)
if err == nil && len(output) == 0 {
if err != nil {
return err
}
if len(output) == 0 {
// Make sure we have a contract to operate on, and bail out otherwise.
if code, err = c.caller.CodeAt(ctx, c.address, opts.BlockNumber); err != nil {
return err

2
accounts/usbwallet/ledger.go
View file

@ -162,7 +162,7 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio
return common.Address{}, nil, accounts.ErrWalletClosed
}
// Ensure the wallet is capable of signing the given transaction
if chainID != nil && w.version[0] <= 1 && w.version[2] <= 2 {
if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
//lint:ignore ST1005 brand name displayed on the console
return common.Address{}, nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
}

6
appveyor.yml
View file

@ -24,13 +24,13 @@ environment:
install:
- git submodule update --init
- rmdir C:\go /s /q
- appveyor DownloadFile https://dl.google.com/go/go1.15.windows-%GETH_ARCH%.zip
- 7z x go1.15.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- appveyor DownloadFile https://dl.google.com/go/go1.15.5.windows-%GETH_ARCH%.zip
- 7z x go1.15.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
- go version
- gcc --version
build_script:
- go run build\ci.go install
- go run build\ci.go install -dlgo
after_build:
- go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds

9
build/checksums.txt
View file

@ -1,6 +1,13 @@
# This file contains sha256 checksums of optional build dependencies.
69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a go1.15.src.tar.gz
c1076b90cf94b73ebed62a81d802cd84d43d02dea8c07abdc922c57a071c84f1 go1.15.5.src.tar.gz
359a4334b8c8f5e3067e5a76f16419791ac3fef4613d8e8e1eac0b9719915f6d go1.15.5.darwin-amd64.tar.gz
4c8179d406136979724c71732009c7e2e7c794dbeaaa2a043c00da34d4be0559 go1.15.5.linux-386.tar.gz
9a58494e8da722c3aef248c9227b0e9c528c7318309827780f16220998180a0d go1.15.5.linux-amd64.tar.gz
a72a0b036beb4193a0214bca3fca4c5d68a38a4ccf098c909f7ce8bf08567c48 go1.15.5.linux-arm64.tar.gz
5ea6456620d3efed5dda99238c7f23866eafdd915e5348736e631bc283c0238a go1.15.5.linux-armv6l.tar.gz
d812436c7e3482ba3c97172edf26afaf35aca60a5621ff4a5f6a08386505ab9c go1.15.5.windows-386.zip
1d24be3a200201a74be25e4134fbec467750e834e84e9c7789a9fc13248c5507 go1.15.5.windows-amd64.zip
d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35 golangci-lint-1.27.0-linux-armv7.tar.gz
bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint.exe-1.27.0-windows-amd64.zip

203
build/ci.go
View file

@ -46,12 +46,11 @@ import (
"encoding/base64"
"flag"
"fmt"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"runtime"
@ -135,11 +134,11 @@ var (
// Note: artful is unsupported because it was officially deprecated on Launchpad.
// Note: cosmic is unsupported because it was officially deprecated on Launchpad.
// Note: disco is unsupported because it was officially deprecated on Launchpad.
// Note: eoan is unsupported because it was officially deprecated on Launchpad.
debDistroGoBoots = map[string]string{
"trusty": "golang-1.11",
"xenial": "golang-go",
"bionic": "golang-go",
"eoan": "golang-go",
"focal": "golang-go",
"groovy": "golang-go",
}
@ -148,6 +147,11 @@ var (
"golang-1.11": "/usr/lib/go-1.11",
"golang-go": "/usr/lib/go",
}
// This is the version of go that will be downloaded by
//
// go run ci.go install -dlgo
dlgoVersion = "1.15.5"
)
var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@ -198,19 +202,19 @@ func main() {
func doInstall(cmdline []string) {
var (
dlgo = flag.Bool("dlgo", false, "Download Go and build with it")
arch = flag.String("arch", "", "Architecture to cross build for")
cc = flag.String("cc", "", "C compiler to cross build with")
)
flag.CommandLine.Parse(cmdline)
env := build.Env()
// Check Go version. People regularly open issues about compilation
// Check local Go version. People regularly open issues about compilation
// failure with outdated Go. This should save them the trouble.
if !strings.Contains(runtime.Version(), "devel") {
// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
var minor int
fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
if minor < 13 {
log.Println("You have Go version", runtime.Version())
log.Println("go-ethereum requires at least Go version 1.13 and cannot")
@ -218,90 +222,110 @@ func doInstall(cmdline []string) {
os.Exit(1)
}
}
// Compile packages given as arguments, or everything if there are no arguments.
packages := []string{"./..."}
if flag.NArg() > 0 {
packages = flag.Args()
// Choose which go command we're going to use.
var gobuild *exec.Cmd
if !*dlgo {
// Default behavior: use the go version which runs ci.go right now.
gobuild = goTool("build")
} else {
// Download of Go requested. This is for build environments where the
// installed version is too old and cannot be upgraded easily.
cachedir := filepath.Join("build", "cache")
goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir)
gobuild = localGoTool(goroot, "build")
}
if *arch == "" || *arch == runtime.GOARCH {
goinstall := goTool("install", buildFlags(env)...)
if runtime.GOARCH == "arm64" {
goinstall.Args = append(goinstall.Args, "-p", "1")
}
goinstall.Args = append(goinstall.Args, "-trimpath")
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
return
// Configure environment for cross build.
if *arch != "" || *arch != runtime.GOARCH {
gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1")
gobuild.Env = append(gobuild.Env, "GOARCH="+*arch)
}
// Seems we are cross compiling, work around forbidden GOBIN
goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
goinstall.Args = append(goinstall.Args, "-trimpath")
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
// Configure C compiler.
if *cc != "" {
gobuild.Env = append(gobuild.Env, "CC="+*cc)
} else if os.Getenv("CC") != "" {
gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC"))
}
if cmds, err := ioutil.ReadDir("cmd"); err == nil {
for _, cmd := range cmds {
pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
if err != nil {
log.Fatal(err)
}
for name := range pkgs {
if name == "main" {
gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
gobuild.Args = append(gobuild.Args, "-v")
gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
build.MustRun(gobuild)
break
}
}
}
// arm64 CI builders are memory-constrained and can't handle concurrent builds,
// better disable it. This check isn't the best, it should probably
// check for something in env instead.
if runtime.GOARCH == "arm64" {
gobuild.Args = append(gobuild.Args, "-p", "1")
}
// Put the default settings in.
gobuild.Args = append(gobuild.Args, buildFlags(env)...)
// We use -trimpath to avoid leaking local paths into the built executables.
gobuild.Args = append(gobuild.Args, "-trimpath")
// Show packages during build.
gobuild.Args = append(gobuild.Args, "-v")
// Now we choose what we're even building.
// Default: collect all 'main' packages in cmd/ and build those.
packages := flag.Args()
if len(packages) == 0 {
packages = build.FindMainPackages("./cmd")
}
// Do the build!
for _, pkg := range packages {
args := make([]string, len(gobuild.Args))
copy(args, gobuild.Args)
args = append(args, "-o", executablePath(path.Base(pkg)))
args = append(args, pkg)
build.MustRun(&exec.Cmd{Path: gobuild.Path, Args: args, Env: gobuild.Env})
}
}
// buildFlags returns the go tool flags for building.
func buildFlags(env build.Environment) (flags []string) {
var ld []string
if env.Commit != "" {
ld = append(ld, "-X", "main.gitCommit="+env.Commit)
ld = append(ld, "-X", "main.gitDate="+env.Date)
}
// Strip DWARF on darwin. This used to be required for certain things,
// and there is no downside to this, so we just keep doing it.
if runtime.GOOS == "darwin" {
ld = append(ld, "-s")
}
if len(ld) > 0 {
flags = append(flags, "-ldflags", strings.Join(ld, " "))
}
return flags
}
// goTool returns the go tool. This uses the Go version which runs ci.go.
func goTool(subcmd string, args ...string) *exec.Cmd {
return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
cmd := build.GoTool(subcmd, args...)
goToolSetEnv(cmd)
return cmd
}
func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
cmd := build.GoTool(subcmd, args...)
if arch == "" || arch == runtime.GOARCH {
cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
} else {
cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
cmd.Env = append(cmd.Env, "GOARCH="+arch)
}
if cc != "" {
cmd.Env = append(cmd.Env, "CC="+cc)
}
// localGoTool returns the go tool from the given GOROOT.
func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd {
gotool := filepath.Join(goroot, "bin", "go")
cmd := exec.Command(gotool, subcmd)
goToolSetEnv(cmd)
cmd.Env = append(cmd.Env, "GOROOT="+goroot)
cmd.Args = append(cmd.Args, args...)
return cmd
}
// goToolSetEnv forwards the build environment to the go tool.
func goToolSetEnv(cmd *exec.Cmd) {
cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
for _, e := range os.Environ() {
if strings.HasPrefix(e, "GOBIN=") {
if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") {
continue
}
cmd.Env = append(cmd.Env, e)
}
return cmd
}
// Running The Tests
@ -363,7 +387,7 @@ func downloadLinter(cachedir string) string {
if err := csdb.DownloadFile(url, archivePath); err != nil {
log.Fatal(err)
}
if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil {
if err := build.ExtractArchive(archivePath, cachedir); err != nil {
log.Fatal(err)
}
return filepath.Join(cachedir, base, "golangci-lint")
@ -469,13 +493,12 @@ func maybeSkipArchive(env build.Environment) {
// Debian Packaging
func doDebianSource(cmdline []string) {
var (
goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
signer = flag.String("signer", "", `Signing key name, also used as package author`)
upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
now = time.Now()
)
flag.CommandLine.Parse(cmdline)
*workdir = makeWorkdir(*workdir)
@ -490,10 +513,10 @@ func doDebianSource(cmdline []string) {
}
// Download and verify the Go source package.
gobundle := downloadGoSources(*goversion, *cachedir)
gobundle := downloadGoSources(*cachedir)
// Download all the dependencies needed to build the sources and run the ci script
srcdepfetch := goTool("install", "-n", "./...")
srcdepfetch := goTool("mod", "download")
srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath"))
build.MustRun(srcdepfetch)
@ -509,7 +532,7 @@ func doDebianSource(cmdline []string) {
pkgdir := stageDebianSource(*workdir, meta)
// Add Go source code
if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil {
if err := build.ExtractArchive(gobundle, pkgdir); err != nil {
log.Fatalf("Failed to extract Go sources: %v", err)
}
if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
@ -541,9 +564,10 @@ func doDebianSource(cmdline []string) {
}
}
func downloadGoSources(version string, cachedir string) string {
// downloadGoSources downloads the Go source tarball.
func downloadGoSources(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
file := fmt.Sprintf("go%s.src.tar.gz", version)
file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
url := "https://dl.google.com/go/" + file
dst := filepath.Join(cachedir, file)
if err := csdb.DownloadFile(url, dst); err != nil {
@ -552,6 +576,41 @@ func downloadGoSources(version string, cachedir string) string {
return dst
}
// downloadGo downloads the Go binary distribution and unpacks it into a temporary
// directory. It returns the GOROOT of the unpacked toolchain.
func downloadGo(goarch, goos, cachedir string) string {
if goarch == "arm" {
goarch = "armv6l"
}
csdb := build.MustLoadChecksums("build/checksums.txt")
file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch)
if goos == "windows" {
file += ".zip"
} else {
file += ".tar.gz"
}
url := "https://golang.org/dl/" + file
dst := filepath.Join(cachedir, file)
if err := csdb.DownloadFile(url, dst); err != nil {
log.Fatal(err)
}
ucache, err := os.UserCacheDir()
if err != nil {
log.Fatal(err)
}
godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch))
if err := build.ExtractArchive(dst, godir); err != nil {
log.Fatal(err)
}
goroot, err := filepath.Abs(filepath.Join(godir, "go"))
if err != nil {
log.Fatal(err)
}
return goroot
}
func ppaUpload(workdir, ppa, sshUser string, files []string) {
p := strings.Split(ppa, "/")
if len(p) != 2 {
@ -980,7 +1039,7 @@ func doXCodeFramework(cmdline []string) {
if *deploy != "" {
meta := newPodMetadata(env, archive)
build.Render("build/pod.podspec", "Geth.podspec", 0755, meta)
build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose")
build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings")
}
}

19
cmd/devp2p/README.md
View file

@ -81,6 +81,25 @@ Now get the ENR of your node and store it in the `NODE` environment variable.
Start the test by running `devp2p discv5 test -listen1 127.0.0.1 -listen2 127.0.0.2 $NODE`.
### Eth Protocol Test Suite
The Eth Protocol test suite is a conformance test suite for the [eth protocol][eth].
To run the eth protocol test suite against your implementation, the node needs to be initialized as such:
1. initialize the geth node with the `genesis.json` file contained in the `testdata` directory
2. import the `halfchain.rlp` file in the `testdata` directory
3. run geth with the following flags:
```
geth --datadir <datadir> --nodiscover --nat=none --networkid 19763 --verbosity 5
```
Then, run the following command, replacing `<enode ID>` with the enode of the geth node:
```
devp2p rlpx eth-test <enode ID> cmd/devp2p/internal/ethtest/testdata/fullchain.rlp cmd/devp2p/internal/ethtest/testdata/genesis.json
```
[eth]: https://github.com/ethereum/devp2p/blob/master/caps/eth.md
[dns-tutorial]: https://geth.ethereum.org/docs/developers/dns-discovery-setup
[discv4]: https://github.com/ethereum/devp2p/tree/master/discv4.md
[discv5]: https://github.com/ethereum/devp2p/tree/master/discv5/discv5.md

38
cmd/devp2p/discv4cmd.go
View file

@ -19,14 +19,12 @@ package main
import (
"fmt"
"net"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
@ -82,7 +80,13 @@ var (
Name: "test",
Usage: "Runs tests against a node",
Action: discv4Test,
Flags: []cli.Flag{remoteEnodeFlag, testPatternFlag, testListen1Flag, testListen2Flag},
Flags: []cli.Flag{
remoteEnodeFlag,
testPatternFlag,
testTAPFlag,
testListen1Flag,
testListen2Flag,
},
}
)
@ -113,20 +117,6 @@ var (
Usage: "Enode of the remote node under test",
EnvVar: "REMOTE_ENODE",
}
testPatternFlag = cli.StringFlag{
Name: "run",
Usage: "Pattern of test suite(s) to run",
}
testListen1Flag = cli.StringFlag{
Name: "listen1",
Usage: "IP address of the first tester",
Value: v4test.Listen1,
}
testListen2Flag = cli.StringFlag{
Name: "listen2",
Usage: "IP address of the second tester",
Value: v4test.Listen2,
}
)
func discv4Ping(ctx *cli.Context) error {
@ -213,6 +203,7 @@ func discv4Crawl(ctx *cli.Context) error {
return nil
}
// discv4Test runs the protocol test suite.
func discv4Test(ctx *cli.Context) error {
// Configure test package globals.
if !ctx.IsSet(remoteEnodeFlag.Name) {
@ -221,18 +212,7 @@ func discv4Test(ctx *cli.Context) error {
v4test.Remote = ctx.String(remoteEnodeFlag.Name)
v4test.Listen1 = ctx.String(testListen1Flag.Name)
v4test.Listen2 = ctx.String(testListen2Flag.Name)
// Filter and run test cases.
tests := v4test.AllTests
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("%v/%v passed\n", len(tests), len(tests))
return nil
return runTests(ctx, v4test.AllTests)
}
// startV4 starts an ephemeral discovery V4 node.

28
cmd/devp2p/discv5cmd.go
View file

@ -18,13 +18,10 @@ package main
import (
"fmt"
"os"
"time"
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"gopkg.in/urfave/cli.v1"
)
@ -62,7 +59,12 @@ var (
Name: "test",
Usage: "Runs protocol tests against a node",
Action: discv5Test,
Flags: []cli.Flag{testPatternFlag, testListen1Flag, testListen2Flag},
Flags: []cli.Flag{
testPatternFlag,
testTAPFlag,
testListen1Flag,
testListen2Flag,
},
}
discv5ListenCommand = cli.Command{
Name: "listen",
@ -114,28 +116,14 @@ func discv5Crawl(ctx *cli.Context) error {
return nil
}
// discv5Test runs the protocol test suite.
func discv5Test(ctx *cli.Context) error {
// Disable logging unless explicitly enabled.
if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
log.Root().SetHandler(log.DiscardHandler())
}
// Filter and run test cases.
suite := &v5test.Suite{
Dest: getNodeArg(ctx),
Listen1: ctx.String(testListen1Flag.Name),
Listen2: ctx.String(testListen2Flag.Name),
}
tests := suite.AllTests()
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("%v/%v passed\n", len(tests), len(tests))
return nil
return runTests(ctx, suite.AllTests())
}
func discv5Listen(ctx *cli.Context) error {

61
cmd/devp2p/internal/ethtest/chain.go
View file

@ -124,32 +124,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) {
// loadChain takes the given chain.rlp file, and decodes and returns
// the blocks from the file.
func loadChain(chainfile string, genesis string) (*Chain, error) {
// Open the file handle and potentially unwrap the gzip stream
fh, err := os.Open(chainfile)
if err != nil {
return nil, err
}
defer fh.Close()
var reader io.Reader = fh
if strings.HasSuffix(chainfile, ".gz") {
if reader, err = gzip.NewReader(reader); err != nil {
return nil, err
}
}
stream := rlp.NewStream(reader, 0)
var blocks []*types.Block
for i := 0; ; i++ {
var b types.Block
if err := stream.Decode(&b); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("at block %d: %v", i, err)
}
blocks = append(blocks, &b)
}
// Open the file handle and potentially unwrap the gzip stream
chainConfig, err := ioutil.ReadFile(genesis)
if err != nil {
return nil, err
@ -158,9 +132,36 @@ func loadChain(chainfile string, genesis string) (*Chain, error) {
if err := json.Unmarshal(chainConfig, &gen); err != nil {
return nil, err
}
gblock := gen.ToBlock(nil)
return &Chain{
blocks: blocks,
chainConfig: gen.Config,
}, nil
// Load chain.rlp.
fh, err := os.Open(chainfile)
if err != nil {
return nil, err
}
defer fh.Close()
var reader io.Reader = fh
if strings.HasSuffix(chainfile, ".gz") {
if reader, err = gzip.NewReader(reader); err != nil {
return nil, err
}
}
stream := rlp.NewStream(reader, 0)
var blocks = make([]*types.Block, 1)
blocks[0] = gblock
for i := 0; ; i++ {
var b types.Block
if err := stream.Decode(&b); err == io.EOF {
break
} else if err != nil {
return nil, fmt.Errorf("at block index %d: %v", i, err)
}
if b.NumberU64() != uint64(i+1) {
return nil, fmt.Errorf("block at index %d has wrong number %d", i, b.NumberU64())
}
blocks = append(blocks, &b)
}
c := &Chain{blocks: blocks, chainConfig: gen.Config}
return c, nil
}

2
cmd/devp2p/internal/ethtest/chain_test.go
View file

@ -73,7 +73,7 @@ func TestEthProtocolNegotiation(t *testing.T) {
// TestChain_GetHeaders tests whether the test suite can correctly
// respond to a GetBlockHeaders request from a node.
func TestChain_GetHeaders(t *testing.T) {
chainFile, err := filepath.Abs("./testdata/chain.rlp.gz")
chainFile, err := filepath.Abs("./testdata/fullchain.rlp.gz")
if err != nil {
t.Fatal(err)
}

55
cmd/devp2p/internal/ethtest/suite.go
View file

@ -19,7 +19,9 @@ package ethtest
import (
"fmt"
"net"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/p2p/enode"
@ -27,6 +29,13 @@ import (
"github.com/stretchr/testify/assert"
)
var pretty = spew.ConfigState{
Indent: " ",
DisableCapacities: true,
DisablePointerAddresses: true,
SortKeys: true,
}
// Suite represents a structure used to test the eth
// protocol of a node(s).
type Suite struct {
@ -73,9 +82,9 @@ func (s *Suite) TestStatus(t *utesting.T) {
// get status
switch msg := conn.statusExchange(t, s.chain).(type) {
case *Status:
t.Logf("%+v\n", msg)
t.Logf("got status message: %s", pretty.Sdump(msg))
default:
t.Fatalf("unexpected: %#v", msg)
t.Fatalf("unexpected: %s", pretty.Sdump(msg))
}
}
@ -104,16 +113,17 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
t.Fatalf("could not write to connection: %v", err)
}
switch msg := conn.ReadAndServe(s.chain).(type) {
timeout := 20 * time.Second
switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
case *BlockHeaders:
headers := msg
for _, header := range *headers {
num := header.Number.Uint64()
t.Logf("received header (%d): %s", num, pretty.Sdump(header))
assert.Equal(t, s.chain.blocks[int(num)].Header(), header)
t.Logf("\nHEADER FOR BLOCK NUMBER %d: %+v\n", header.Number, header)
}
default:
t.Fatalf("unexpected: %#v", msg)
t.Fatalf("unexpected: %s", pretty.Sdump(msg))
}
}
@ -133,14 +143,12 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
t.Fatalf("could not write to connection: %v", err)
}
switch msg := conn.ReadAndServe(s.chain).(type) {
timeout := 20 * time.Second
switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
case *BlockBodies:
bodies := msg
for _, body := range *bodies {
t.Logf("\nBODY: %+v\n", body)
}
t.Logf("received %d block bodies", len(*msg))
default:
t.Fatalf("unexpected: %#v", msg)
t.Fatalf("unexpected: %s", pretty.Sdump(msg))
}
}
@ -173,18 +181,27 @@ func (s *Suite) TestBroadcast(t *utesting.T) {
t.Fatalf("could not write to connection: %v", err)
}
switch msg := receiveConn.ReadAndServe(s.chain).(type) {
timeout := 20 * time.Second
switch msg := receiveConn.ReadAndServe(s.chain, timeout).(type) {
case *NewBlock:
assert.Equal(t, blockAnnouncement.Block.Header(), msg.Block.Header(),
"wrong block header in announcement")
assert.Equal(t, blockAnnouncement.TD, msg.TD,
"wrong TD in announcement")
t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block))
assert.Equal(t,
blockAnnouncement.Block.Header(), msg.Block.Header(),
"wrong block header in announcement",
)
assert.Equal(t,
blockAnnouncement.TD, msg.TD,
"wrong TD in announcement",
)
case *NewBlockHashes:
hashes := *msg
assert.Equal(t, blockAnnouncement.Block.Hash(), hashes[0].Hash,
"wrong block hash in announcement")
t.Logf("received NewBlockHashes message: %s", pretty.Sdump(hashes))
assert.Equal(t,
blockAnnouncement.Block.Hash(), hashes[0].Hash,
"wrong block hash in announcement",
)
default:
t.Fatalf("unexpected: %#v", msg)
t.Fatalf("unexpected: %s", pretty.Sdump(msg))
}
// update test suite chain
s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[1000])

BIN
cmd/devp2p/internal/ethtest/testdata/fullchain.rlp.gz vendored Normal file
View file

Binary file not shown.

10
cmd/devp2p/internal/ethtest/testdata/genesis.json vendored
View file

@ -1,6 +1,6 @@
{
"config": {
"chainId": 1,
"chainId": 19763,
"homesteadBlock": 0,
"eip150Block": 0,
"eip155Block": 0,
@ -11,16 +11,16 @@
"nonce": "0xdeadbeefdeadbeef",
"timestamp": "0x0",
"extraData": "0x0000000000000000000000000000000000000000000000000000000000000000",
"gasLimit": "0x8000000",
"difficulty": "0x10",
"gasLimit": "0x80000000",
"difficulty": "0x20000",
"mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
"coinbase": "0x0000000000000000000000000000000000000000",
"alloc": {
"71562b71999873db5b286df957af199ec94617f7": {
"balance": "0xf4240"
"balance": "0xffffffff"
}
},
"number": "0x0",
"gasUsed": "0x0",
"parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000"
}
}

BIN
cmd/devp2p/internal/ethtest/testdata/halfchain.rlp.gz vendored Normal file
View file

Binary file not shown.

57
cmd/devp2p/internal/ethtest/types.go
View file

@ -42,10 +42,14 @@ type Error struct {
err error
}
func (e *Error) Unwrap() error { return e.err }
func (e *Error) Error() string { return e.err.Error() }
func (e *Error) Code() int { return -1 }
func (e *Error) GoString() string { return e.Error() }
func (e *Error) Unwrap() error { return e.err }
func (e *Error) Error() string { return e.err.Error() }
func (e *Error) Code() int { return -1 }
func (e *Error) String() string { return e.Error() }
func errorf(format string, args ...interface{}) *Error {
return &Error{fmt.Errorf(format, args...)}
}
// Hello is the RLP structure of the protocol handshake.
type Hello struct {
@ -174,7 +178,7 @@ type Conn struct {
func (c *Conn) Read() Message {
code, rawData, _, err := c.Conn.Read()
if err != nil {
return &Error{fmt.Errorf("could not read from connection: %v", err)}
return errorf("could not read from connection: %v", err)
}
var msg Message
@ -202,20 +206,22 @@ func (c *Conn) Read() Message {
case (NewBlockHashes{}).Code():
msg = new(NewBlockHashes)
default:
return &Error{fmt.Errorf("invalid message code: %d", code)}
return errorf("invalid message code: %d", code)
}
if err := rlp.DecodeBytes(rawData, msg); err != nil {
return &Error{fmt.Errorf("could not rlp decode message: %v", err)}
return errorf("could not rlp decode message: %v", err)
}
return msg
}
// ReadAndServe serves GetBlockHeaders requests while waiting
// on another message from the node.
func (c *Conn) ReadAndServe(chain *Chain) Message {
for {
func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message {
start := time.Now()
for time.Since(start) < timeout {
timeout := time.Now().Add(10 * time.Second)
c.SetReadDeadline(timeout)
switch msg := c.Read().(type) {
case *Ping:
c.Write(&Pong{})
@ -223,16 +229,17 @@ func (c *Conn) ReadAndServe(chain *Chain) Message {
req := *msg
headers, err := chain.GetHeaders(req)
if err != nil {
return &Error{fmt.Errorf("could not get headers for inbound header request: %v", err)}
return errorf("could not get headers for inbound header request: %v", err)
}
if err := c.Write(headers); err != nil {
return &Error{fmt.Errorf("could not write to connection: %v", err)}
return errorf("could not write to connection: %v", err)
}
default:
return msg
}
}
return errorf("no message received within %v", timeout)
}
func (c *Conn) Write(msg Message) error {
@ -242,12 +249,14 @@ func (c *Conn) Write(msg Message) error {
}
_, err = c.Conn.Write(uint64(msg.Code()), payload)
return err
}
// handshake checks to make sure a `HELLO` is received.
func (c *Conn) handshake(t *utesting.T) Message {
// write protoHandshake to client
defer c.SetDeadline(time.Time{})
c.SetDeadline(time.Now().Add(10 * time.Second))
// write hello to client
pub0 := crypto.FromECDSAPub(&c.ourKey.PublicKey)[1:]
ourHandshake := &Hello{
Version: 5,
@ -260,14 +269,13 @@ func (c *Conn) handshake(t *utesting.T) Message {
if err := c.Write(ourHandshake); err != nil {
t.Fatalf("could not write to connection: %v", err)
}
// read protoHandshake from client
// read hello from client
switch msg := c.Read().(type) {
case *Hello:
// set snappy if version is at least 5
if msg.Version >= 5 {
c.SetSnappy(true)
}
c.negotiateEthProtocol(msg.Caps)
if c.ethProtocolVersion == 0 {
t.Fatalf("unexpected eth protocol version")
@ -297,15 +305,17 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
// statusExchange performs a `Status` message exchange with the given
// node.
func (c *Conn) statusExchange(t *utesting.T, chain *Chain) Message {
defer c.SetDeadline(time.Time{})
c.SetDeadline(time.Now().Add(20 * time.Second))
// read status message from client
var message Message
loop:
for {
switch msg := c.Read().(type) {
case *Status:
if msg.Head != chain.blocks[chain.Len()-1].Hash() {
t.Fatalf("wrong head in status: %v", msg.Head)
t.Fatalf("wrong head block in status: %s", msg.Head.String())
}
if msg.TD.Cmp(chain.TD(chain.Len())) != 0 {
t.Fatalf("wrong TD in status: %v", msg.TD)
@ -321,7 +331,7 @@ loop:
c.Write(&Pong{}) // TODO (renaynay): in the future, this should be an error
// (PINGs should not be a response upon fresh connection)
default:
t.Fatalf("bad status message: %#v", msg)
t.Fatalf("bad status message: %s", pretty.Sdump(msg))
}
}
// make sure eth protocol version is set for negotiation
@ -331,7 +341,7 @@ loop:
// write status message to client
status := Status{
ProtocolVersion: uint32(c.ethProtocolVersion),
NetworkID: 1,
NetworkID: chain.chainConfig.ChainID.Uint64(),
TD: chain.TD(chain.Len()),
Head: chain.blocks[chain.Len()-1].Hash(),
Genesis: chain.blocks[0].Hash(),
@ -347,12 +357,15 @@ loop:
// waitForBlock waits for confirmation from the client that it has
// imported the given block.
func (c *Conn) waitForBlock(block *types.Block) error {
defer c.SetReadDeadline(time.Time{})
timeout := time.Now().Add(20 * time.Second)
c.SetReadDeadline(timeout)
for {
req := &GetBlockHeaders{Origin: hashOrNumber{Hash: block.Hash()}, Amount: 1}
if err := c.Write(req); err != nil {
return err
}
switch msg := c.Read().(type) {
case *BlockHeaders:
if len(*msg) > 0 {
@ -360,7 +373,7 @@ func (c *Conn) waitForBlock(block *types.Block) error {
}
time.Sleep(100 * time.Millisecond)
default:
return fmt.Errorf("invalid message: %v", msg)
return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))
}
}
}

24
cmd/devp2p/rlpxcmd.go
View file

@ -19,11 +19,9 @@ package main
import (
"fmt"
"net"
"os"
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/rlpx"
"github.com/ethereum/go-ethereum/rlp"
@ -47,9 +45,12 @@ var (
rlpxEthTestCommand = cli.Command{
Name: "eth-test",
Usage: "Runs tests against a node",
ArgsUsage: "<node> <path_to_chain.rlp_file>",
ArgsUsage: "<node> <chain.rlp> <genesis.json>",
Action: rlpxEthTest,
Flags: []cli.Flag{testPatternFlag},
Flags: []cli.Flag{
testPatternFlag,
testTAPFlag,
},
}
)
@ -88,22 +89,11 @@ func rlpxPing(ctx *cli.Context) error {
return nil
}
// rlpxEthTest runs the eth protocol test suite.
func rlpxEthTest(ctx *cli.Context) error {
if ctx.NArg() < 3 {
exit("missing path to chain.rlp as command-line argument")
}
suite := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])
// Filter and run test cases.
tests := suite.AllTests()
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v of %v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("all tests passed\n")
return nil
return runTests(ctx, suite.AllTests())
}

69
cmd/devp2p/runtest.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"os"
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
"gopkg.in/urfave/cli.v1"
)
var (
testPatternFlag = cli.StringFlag{
Name: "run",
Usage: "Pattern of test suite(s) to run",
}
testTAPFlag = cli.BoolFlag{
Name: "tap",
Usage: "Output TAP",
}
// These two are specific to the discovery tests.
testListen1Flag = cli.StringFlag{
Name: "listen1",
Usage: "IP address of the first tester",
Value: v4test.Listen1,
}
testListen2Flag = cli.StringFlag{
Name: "listen2",
Usage: "IP address of the second tester",
Value: v4test.Listen2,
}
)
func runTests(ctx *cli.Context, tests []utesting.Test) error {
// Filter test cases.
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
// Disable logging unless explicitly enabled.
if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
log.Root().SetHandler(log.DiscardHandler())
}
// Run the tests.
var run = utesting.RunTests
if ctx.Bool(testTAPFlag.Name) {
run = utesting.RunTAP
}
results := run(tests, os.Stdout)
if utesting.CountFailures(results) > 0 {
os.Exit(1)
}
return nil
}

10
cmd/evm/internal/t8ntool/execution.go
View file

@ -147,6 +147,16 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
vmContext.Origin = msg.From()
evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig)
if chainConfig.IsYoloV2(vmContext.BlockNumber) {
statedb.AddAddressToAccessList(msg.From())
if dst := msg.To(); dst != nil {
statedb.AddAddressToAccessList(*dst)
// If it's a create-tx, the destination will be added inside evm.create
}
for _, addr := range evm.ActivePrecompiles() {
statedb.AddAddressToAccessList(addr)
}
}
snapshot := statedb.Snapshot()
// (ret []byte, usedGas uint64, failed bool, err error)
msgResult, err := core.ApplyMessage(evm, msg, gaspool)

4
cmd/geth/chaincmd.go
View file

@ -163,7 +163,7 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
utils.RinkebyFlag,
utils.TxLookupLimitFlag,
utils.GoerliFlag,
utils.YoloV1Flag,
utils.YoloV2Flag,
utils.LegacyTestnetFlag,
},
Category: "BLOCKCHAIN COMMANDS",
@ -213,7 +213,7 @@ Use "ethereum dump 0" to dump the genesis block.`,
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV1Flag,
utils.YoloV2Flag,
utils.LegacyTestnetFlag,
utils.SyncModeFlag,
},

4
cmd/geth/consolecmd.go
View file

@ -136,8 +136,8 @@ func remoteConsole(ctx *cli.Context) error {
path = filepath.Join(path, "rinkeby")
} else if ctx.GlobalBool(utils.GoerliFlag.Name) {
path = filepath.Join(path, "goerli")
} else if ctx.GlobalBool(utils.YoloV1Flag.Name) {
path = filepath.Join(path, "yolo-v1")
} else if ctx.GlobalBool(utils.YoloV2Flag.Name) {
path = filepath.Join(path, "yolo-v2")
}
}
endpoint = fmt.Sprintf("%s/geth.ipc", path)

2
cmd/geth/consolecmd_test.go
View file

@ -66,6 +66,7 @@ at block: 0 ({{niltime}})
datadir: {{.Datadir}}
modules: {{apis}}
To exit, press ctrl-d
> {{.InputLine "exit"}}
`)
geth.ExpectExit()
@ -159,6 +160,7 @@ at block: 0 ({{niltime}}){{if ipc}}
datadir: {{datadir}}{{end}}
modules: {{apis}}
To exit, press ctrl-d
> {{.InputLine "exit" }}
`)
attach.ExpectExit()

2
cmd/geth/main.go
View file

@ -144,7 +144,7 @@ var (
utils.RopstenFlag,
utils.RinkebyFlag,
utils.GoerliFlag,
utils.YoloV1Flag,
utils.YoloV2Flag,
utils.VMEnableDebugFlag,
utils.NetworkIdFlag,
utils.EthStatsURLFlag,

2
cmd/geth/usage.go
View file

@ -42,7 +42,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
utils.NetworkIdFlag,
utils.GoerliFlag,
utils.RinkebyFlag,
utils.YoloV1Flag,
utils.YoloV2Flag,
utils.RopstenFlag,
utils.SyncModeFlag,
utils.ExitWhenSyncedFlag,

4
cmd/puppeth/wizard_genesis.go
View file

@ -236,8 +236,8 @@ func (w *wizard) manageGenesis() {
w.conf.Genesis.Config.IstanbulBlock = w.readDefaultBigInt(w.conf.Genesis.Config.IstanbulBlock)
fmt.Println()
fmt.Printf("Which block should YOLOv1 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV1Block)
w.conf.Genesis.Config.YoloV1Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV1Block)
fmt.Printf("Which block should YOLOv2 come into effect? (default = %v)\n", w.conf.Genesis.Config.YoloV2Block)
w.conf.Genesis.Config.YoloV2Block = w.readDefaultBigInt(w.conf.Genesis.Config.YoloV2Block)
out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ")
fmt.Printf("Chain configuration updated:\n\n%s\n", out)

34
cmd/utils/flags.go
View file

@ -135,9 +135,9 @@ var (
Name: "goerli",
Usage: "Görli network: pre-configured proof-of-authority test network",
}
YoloV1Flag = cli.BoolFlag{
Name: "yolov1",
Usage: "YOLOv1 network: pre-configured proof-of-authority shortlived test network.",
YoloV2Flag = cli.BoolFlag{
Name: "yolov2",
Usage: "YOLOv2 network: pre-configured proof-of-authority shortlived test network.",
}
RinkebyFlag = cli.BoolFlag{
Name: "rinkeby",
@ -744,8 +744,8 @@ func MakeDataDir(ctx *cli.Context) string {
if ctx.GlobalBool(GoerliFlag.Name) {
return filepath.Join(path, "goerli")
}
if ctx.GlobalBool(YoloV1Flag.Name) {
return filepath.Join(path, "yolo-v1")
if ctx.GlobalBool(YoloV2Flag.Name) {
return filepath.Join(path, "yolo-v2")
}
return path
}
@ -803,8 +803,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
urls = params.GoerliBootnodes
case ctx.GlobalBool(YoloV1Flag.Name):
urls = params.YoloV1Bootnodes
case ctx.GlobalBool(YoloV2Flag.Name):
urls = params.YoloV2Bootnodes
case cfg.BootstrapNodes != nil:
return // already set, don't apply defaults.
}
@ -839,8 +839,8 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) {
urls = params.RinkebyBootnodes
case ctx.GlobalBool(GoerliFlag.Name):
urls = params.GoerliBootnodes
case ctx.GlobalBool(YoloV1Flag.Name):
urls = params.YoloV1Bootnodes
case ctx.GlobalBool(YoloV2Flag.Name):
urls = params.YoloV2Bootnodes
case cfg.BootstrapNodesV5 != nil:
return // already set, don't apply defaults.
}
@ -1269,8 +1269,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
case ctx.GlobalBool(YoloV1Flag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v1")
case ctx.GlobalBool(YoloV2Flag.Name) && cfg.DataDir == node.DefaultDataDir():
cfg.DataDir = filepath.Join(node.DefaultDataDir(), "yolo-v2")
}
}
@ -1483,7 +1483,7 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
// Avoid conflicting network flags
CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV1Flag)
CheckExclusive(ctx, DeveloperFlag, LegacyTestnetFlag, RopstenFlag, RinkebyFlag, GoerliFlag, YoloV2Flag)
CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, SyncModeFlag, "light")
CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
CheckExclusive(ctx, GCModeFlag, "archive", TxLookupLimitFlag)
@ -1603,11 +1603,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
}
cfg.Genesis = core.DefaultGoerliGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash)
case ctx.GlobalBool(YoloV1Flag.Name):
case ctx.GlobalBool(YoloV2Flag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 133519467574833 // "yolov1"
cfg.NetworkId = 133519467574834 // "yolov2"
}
cfg.Genesis = core.DefaultYoloV1GenesisBlock()
cfg.Genesis = core.DefaultYoloV2GenesisBlock()
case ctx.GlobalBool(DeveloperFlag.Name):
if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 1337
@ -1791,8 +1791,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
genesis = core.DefaultRinkebyGenesisBlock()
case ctx.GlobalBool(GoerliFlag.Name):
genesis = core.DefaultGoerliGenesisBlock()
case ctx.GlobalBool(YoloV1Flag.Name):
genesis = core.DefaultYoloV1GenesisBlock()
case ctx.GlobalBool(YoloV2Flag.Name):
genesis = core.DefaultYoloV2GenesisBlock()
case ctx.GlobalBool(DeveloperFlag.Name):
Fatalf("Developer chains are ephemeral")
}

25
common/bytes.go
View file

@ -17,28 +17,9 @@
// Package common contains various helper functions.
package common
import "encoding/hex"
// ToHex returns the hex representation of b, prefixed with '0x'.
// For empty slices, the return value is "0x0".
//
// Deprecated: use hexutil.Encode instead.
func ToHex(b []byte) string {
hex := Bytes2Hex(b)
if len(hex) == 0 {
hex = "0"
}
return "0x" + hex
}
// ToHexArray creates a array of hex-string based on []byte
func ToHexArray(b [][]byte) []string {
r := make([]string, len(b))
for i := range b {
r[i] = ToHex(b[i])
}
return r
}
import (
"encoding/hex"
)
// FromHex returns the bytes represented by the hexadecimal string s.
// s may be prefixed with "0x".

18
consensus/ethash/algorithm.go
View file

@ -295,7 +295,7 @@ func generateDataset(dest []uint32, epoch uint64, cache []uint32) {
var pend sync.WaitGroup
pend.Add(threads)
var progress uint32
var progress uint64
for i := 0; i < threads; i++ {
go func(id int) {
defer pend.Done()
@ -304,23 +304,23 @@ func generateDataset(dest []uint32, epoch uint64, cache []uint32) {
keccak512 := makeHasher(sha3.NewLegacyKeccak512())
// Calculate the data segment this thread should generate
batch := uint32((size + hashBytes*uint64(threads) - 1) / (hashBytes * uint64(threads)))
first := uint32(id) * batch
batch := (size + hashBytes*uint64(threads) - 1) / (hashBytes * uint64(threads))
first := uint64(id) * batch
limit := first + batch
if limit > uint32(size/hashBytes) {
limit = uint32(size / hashBytes)
if limit > size/hashBytes {
limit = size / hashBytes
}
// Calculate the dataset segment
percent := uint32(size / hashBytes / 100)
percent := size / hashBytes / 100
for index := first; index < limit; index++ {
item := generateDatasetItem(cache, index, keccak512)
item := generateDatasetItem(cache, uint32(index), keccak512)
if swapped {
swap(item)
}
copy(dataset[index*hashBytes:], item)
if status := atomic.AddUint32(&progress, 1); status%percent == 0 {
logger.Info("Generating DAG in progress", "percentage", uint64(status*100)/(size/hashBytes), "elapsed", common.PrettyDuration(time.Since(start)))
if status := atomic.AddUint64(&progress, 1); status%percent == 0 {
logger.Info("Generating DAG in progress", "percentage", (status*100)/(size/hashBytes), "elapsed", common.PrettyDuration(time.Since(start)))
}
}
}(i)

3
console/console.go
View file

@ -324,6 +324,7 @@ func (c *Console) Welcome() {
sort.Strings(modules)
message += " modules: " + strings.Join(modules, " ") + "\n"
}
message += "\nTo exit, press ctrl-d"
fmt.Fprintln(c.printer, message)
}
@ -372,7 +373,7 @@ func (c *Console) Interactive() {
return
case err := <-inputErr:
if err == liner.ErrPromptAborted && indents > 0 {
if err == liner.ErrPromptAborted {
// When prompting for multi-line input, the first Ctrl-C resets
// the multi-line state.
prompt, indents, input = c.prompt, 0, ""

124
core/blockchain.go
View file

@ -207,9 +207,10 @@ type BlockChain struct {
processor Processor // Block transaction processor interface
vmConfig vm.Config
badBlocks *lru.Cache // Bad block cache
shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion.
badBlocks *lru.Cache // Bad block cache
shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion.
writeLegacyJournal bool // Testing flag used to flush the snapshot journal in legacy format.
}
// NewBlockChain returns a fully initialised block chain using information
@ -281,9 +282,29 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
// Make sure the state associated with the block is available
head := bc.CurrentBlock()
if _, err := state.New(head.Root(), bc.stateCache, bc.snaps); err != nil {
log.Warn("Head state missing, repairing", "number", head.Number(), "hash", head.Hash())
if err := bc.SetHead(head.NumberU64()); err != nil {
return nil, err
// Head state is missing, before the state recovery, find out the
// disk layer point of snapshot(if it's enabled). Make sure the
// rewound point is lower than disk layer.
var diskRoot common.Hash
if bc.cacheConfig.SnapshotLimit > 0 {
diskRoot = rawdb.ReadSnapshotRoot(bc.db)
}
if diskRoot != (common.Hash{}) {
log.Warn("Head state missing, repairing", "number", head.Number(), "hash", head.Hash(), "snaproot", diskRoot)
snapDisk, err := bc.SetHeadBeyondRoot(head.NumberU64(), diskRoot)
if err != nil {
return nil, err
}
// Chain rewound, persist old snapshot number to indicate recovery procedure
if snapDisk != 0 {
rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
}
} else {
log.Warn("Head state missing, repairing", "number", head.Number(), "hash", head.Hash())
if err := bc.SetHead(head.NumberU64()); err != nil {
return nil, err
}
}
}
// Ensure that a previous crash in SetHead doesn't leave extra ancients
@ -339,12 +360,25 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
}
// Load any existing snapshot, regenerating it if loading failed
if bc.cacheConfig.SnapshotLimit > 0 {
bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, bc.CurrentBlock().Root(), !bc.cacheConfig.SnapshotWait)
// If the chain was rewound past the snapshot persistent layer (causing
// a recovery block number to be persisted to disk), check if we're still
// in recovery mode and in that case, don't invalidate the snapshot on a
// head mismatch.
var recover bool
head := bc.CurrentBlock()
if layer := rawdb.ReadSnapshotRecoveryNumber(bc.db); layer != nil && *layer > head.NumberU64() {
log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
recover = true
}
bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, recover)
}
// Take ownership of this particular state
go bc.update()
if txLookupLimit != nil {
bc.txLookupLimit = *txLookupLimit
bc.wg.Add(1)
go bc.maintainTxIndex(txIndexBlock)
}
// If periodic cache journal is required, spin it up.
@ -442,9 +476,25 @@ func (bc *BlockChain) loadLastState() error {
// was fast synced or full synced and in which state, the method will try to
// delete minimal data from disk whilst retaining chain consistency.
func (bc *BlockChain) SetHead(head uint64) error {
_, err := bc.SetHeadBeyondRoot(head, common.Hash{})
return err
}
// SetHeadBeyondRoot rewinds the local chain to a new head with the extra condition
// that the rewind must pass the specified state root. This method is meant to be
// used when rewiding with snapshots enabled to ensure that we go back further than
// persistent disk layer. Depending on whether the node was fast synced or full, and
// in which state, the method will try to delete minimal data from disk whilst
// retaining chain consistency.
//
// The method returns the block number where the requested root cap was found.
func (bc *BlockChain) SetHeadBeyondRoot(head uint64, root common.Hash) (uint64, error) {
bc.chainmu.Lock()
defer bc.chainmu.Unlock()
// Track the block number of the requested root hash
var rootNumber uint64 // (no root == always 0)
// Retrieve the last pivot block to short circuit rollbacks beyond it and the
// current freezer limit to start nuking id underflown
pivot := rawdb.ReadLastPivotNumber(bc.db)
@ -460,8 +510,16 @@ func (bc *BlockChain) SetHead(head uint64) error {
log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash())
newHeadBlock = bc.genesisBlock
} else {
// Block exists, keep rewinding until we find one with state
// Block exists, keep rewinding until we find one with state,
// keeping rewinding until we exceed the optional threshold
// root hash
beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true)
for {
// If a root threshold was requested but not yet crossed, check
if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root {
beyondRoot, rootNumber = true, newHeadBlock.NumberU64()
}
if _, err := state.New(newHeadBlock.Root(), bc.stateCache, bc.snaps); err != nil {
log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
if pivot == nil || newHeadBlock.NumberU64() > *pivot {
@ -472,8 +530,12 @@ func (bc *BlockChain) SetHead(head uint64) error {
newHeadBlock = bc.genesisBlock
}
}
log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
break
if beyondRoot || newHeadBlock.NumberU64() == 0 {
log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
break
}
log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root())
newHeadBlock = bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1) // Keep rewinding
}
}
rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash())
@ -553,7 +615,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
bc.txLookupCache.Purge()
bc.futureBlocks.Purge()
return bc.loadLastState()
return rootNumber, bc.loadLastState()
}
// FastSyncCommitHead sets the current head block to the one defined by the hash
@ -938,8 +1000,14 @@ func (bc *BlockChain) Stop() {
var snapBase common.Hash
if bc.snaps != nil {
var err error
if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil {
log.Error("Failed to journal state snapshot", "err", err)
if bc.writeLegacyJournal {
if snapBase, err = bc.snaps.LegacyJournal(bc.CurrentBlock().Root()); err != nil {
log.Error("Failed to journal state snapshot", "err", err)
}
} else {
if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil {
log.Error("Failed to journal state snapshot", "err", err)
}
}
}
// Ensure the state of a recent block is also stored to disk before exiting.
@ -2096,6 +2164,20 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
}
return ret
}
// mergeLogs returns a merged log slice with specified sort order.
mergeLogs = func(logs [][]*types.Log, reverse bool) []*types.Log {
var ret []*types.Log
if reverse {
for i := len(logs) - 1; i >= 0; i-- {
ret = append(ret, logs[i]...)
}
} else {
for i := 0; i < len(logs); i++ {
ret = append(ret, logs[i]...)
}
}
return ret
}
)
// Reduce the longer chain to the same number as the shorter one
if oldBlock.NumberU64() > newBlock.NumberU64() {
@ -2230,6 +2312,8 @@ func (bc *BlockChain) update() {
// sync, Geth will automatically construct the missing indices and delete
// the extra indices.
func (bc *BlockChain) maintainTxIndex(ancients uint64) {
defer bc.wg.Done()
// Before starting the actual maintenance, we need to handle a special case,
// where user might init Geth with an external ancient database. If so, we
// need to reindex all necessary transactions before starting to process any
@ -2239,7 +2323,7 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
if bc.txLookupLimit != 0 && ancients > bc.txLookupLimit {
from = ancients - bc.txLookupLimit
}
rawdb.IndexTransactions(bc.db, from, ancients)
rawdb.IndexTransactions(bc.db, from, ancients, bc.quit)
}
// indexBlocks reindexes or unindexes transactions depending on user configuration
indexBlocks := func(tail *uint64, head uint64, done chan struct{}) {
@ -2253,24 +2337,24 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
rawdb.WriteTxIndexTail(bc.db, 0)
} else {
// Prune all stale tx indices and record the tx index tail
rawdb.UnindexTransactions(bc.db, 0, head-bc.txLookupLimit+1)
rawdb.UnindexTransactions(bc.db, 0, head-bc.txLookupLimit+1, bc.quit)
}
return
}
// If a previous indexing existed, make sure that we fill in any missing entries
if bc.txLookupLimit == 0 || head < bc.txLookupLimit {
if *tail > 0 {
rawdb.IndexTransactions(bc.db, 0, *tail)
rawdb.IndexTransactions(bc.db, 0, *tail, bc.quit)
}
return
}
// Update the transaction index to the new chain state
if head-bc.txLookupLimit+1 < *tail {
// Reindex a part of missing indices and rewind index tail to HEAD-limit
rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail)
rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail, bc.quit)
} else {
// Unindex a part of stale indices and forward index tail to HEAD-limit
rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1)
rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1, bc.quit)
}
}
// Any reindexing done, start listening to chain events and moving the index window
@ -2294,6 +2378,10 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) {
case <-done:
done = nil
case <-bc.quit:
if done != nil {
log.Info("Waiting background transaction indexer to exit")
<-done
}
return
}
}

2
core/blockchain_insert.go
View file

@ -43,7 +43,7 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor
// Fetch the timings for the batch
var (
now = mclock.Now()
elapsed = time.Duration(now) - time.Duration(st.startTime)
elapsed = now.Sub(st.startTime)
)
// If we're at the last block of the batch or report period reached, log
if index == len(chain)-1 || elapsed >= statsReportLimit {

318
core/blockchain_repair_test.go
View file

@ -25,6 +25,7 @@ import (
"math/big"
"os"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
@ -38,7 +39,10 @@ import (
// committed to disk and then the process crashed. In this case we expect the full
// chain to be rolled back to the committed block, but the chain data itself left
// in the database for replaying.
func TestShortRepair(t *testing.T) {
func TestShortRepair(t *testing.T) { testShortRepair(t, false) }
func TestShortRepairWithSnapshots(t *testing.T) { testShortRepair(t, true) }
func testShortRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -68,14 +72,17 @@ func TestShortRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain where the fast sync pivot point was
// already committed, after which the process crashed. In this case we expect the full
// chain to be rolled back to the committed block, but the chain data itself left in
// the database for replaying.
func TestShortFastSyncedRepair(t *testing.T) {
func TestShortFastSyncedRepair(t *testing.T) { testShortFastSyncedRepair(t, false) }
func TestShortFastSyncedRepairWithSnapshots(t *testing.T) { testShortFastSyncedRepair(t, true) }
func testShortFastSyncedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -105,14 +112,17 @@ func TestShortFastSyncedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain where the fast sync pivot point was
// not yet committed, but the process crashed. In this case we expect the chain to
// detect that it was fast syncing and not delete anything, since we can just pick
// up directly where we left off.
func TestShortFastSyncingRepair(t *testing.T) {
func TestShortFastSyncingRepair(t *testing.T) { testShortFastSyncingRepair(t, false) }
func TestShortFastSyncingRepairWithSnapshots(t *testing.T) { testShortFastSyncingRepair(t, true) }
func testShortFastSyncingRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -142,7 +152,7 @@ func TestShortFastSyncingRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where a
@ -150,7 +160,10 @@ func TestShortFastSyncingRepair(t *testing.T) {
// test scenario the side chain is below the committed block. In this case we expect
// the canonical chain to be rolled back to the committed block, but the chain data
// itself left in the database for replaying.
func TestShortOldForkedRepair(t *testing.T) {
func TestShortOldForkedRepair(t *testing.T) { testShortOldForkedRepair(t, false) }
func TestShortOldForkedRepairWithSnapshots(t *testing.T) { testShortOldForkedRepair(t, true) }
func testShortOldForkedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -182,7 +195,7 @@ func TestShortOldForkedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where
@ -191,6 +204,13 @@ func TestShortOldForkedRepair(t *testing.T) {
// this case we expect the canonical chain to be rolled back to the committed block,
// but the chain data itself left in the database for replaying.
func TestShortOldForkedFastSyncedRepair(t *testing.T) {
testShortOldForkedFastSyncedRepair(t, false)
}
func TestShortOldForkedFastSyncedRepairWithSnapshots(t *testing.T) {
testShortOldForkedFastSyncedRepair(t, true)
}
func testShortOldForkedFastSyncedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -222,7 +242,7 @@ func TestShortOldForkedFastSyncedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where
@ -231,6 +251,13 @@ func TestShortOldForkedFastSyncedRepair(t *testing.T) {
// the chain to detect that it was fast syncing and not delete anything, since we
// can just pick up directly where we left off.
func TestShortOldForkedFastSyncingRepair(t *testing.T) {
testShortOldForkedFastSyncingRepair(t, false)
}
func TestShortOldForkedFastSyncingRepairWithSnapshots(t *testing.T) {
testShortOldForkedFastSyncingRepair(t, true)
}
func testShortOldForkedFastSyncingRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -262,7 +289,7 @@ func TestShortOldForkedFastSyncingRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where a
@ -270,7 +297,10 @@ func TestShortOldForkedFastSyncingRepair(t *testing.T) {
// test scenario the side chain reaches above the committed block. In this case we
// expect the canonical chain to be rolled back to the committed block, but the
// chain data itself left in the database for replaying.
func TestShortNewlyForkedRepair(t *testing.T) {
func TestShortNewlyForkedRepair(t *testing.T) { testShortNewlyForkedRepair(t, false) }
func TestShortNewlyForkedRepairWithSnapshots(t *testing.T) { testShortNewlyForkedRepair(t, true) }
func testShortNewlyForkedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6
@ -302,7 +332,7 @@ func TestShortNewlyForkedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where
@ -311,6 +341,13 @@ func TestShortNewlyForkedRepair(t *testing.T) {
// In this case we expect the canonical chain to be rolled back to the committed
// block, but the chain data itself left in the database for replaying.
func TestShortNewlyForkedFastSyncedRepair(t *testing.T) {
testShortNewlyForkedFastSyncedRepair(t, false)
}
func TestShortNewlyForkedFastSyncedRepairWithSnapshots(t *testing.T) {
testShortNewlyForkedFastSyncedRepair(t, true)
}
func testShortNewlyForkedFastSyncedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6
@ -342,7 +379,7 @@ func TestShortNewlyForkedFastSyncedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a shorter side chain, where
@ -351,6 +388,13 @@ func TestShortNewlyForkedFastSyncedRepair(t *testing.T) {
// case we expect the chain to detect that it was fast syncing and not delete
// anything, since we can just pick up directly where we left off.
func TestShortNewlyForkedFastSyncingRepair(t *testing.T) {
testShortNewlyForkedFastSyncingRepair(t, false)
}
func TestShortNewlyForkedFastSyncingRepairWithSnapshots(t *testing.T) {
testShortNewlyForkedFastSyncingRepair(t, true)
}
func testShortNewlyForkedFastSyncingRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6
@ -382,14 +426,17 @@ func TestShortNewlyForkedFastSyncingRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a longer side chain, where a
// recent block was already committed to disk and then the process crashed. In this
// case we expect the canonical chain to be rolled back to the committed block, but
// the chain data itself left in the database for replaying.
func TestShortReorgedRepair(t *testing.T) {
func TestShortReorgedRepair(t *testing.T) { testShortReorgedRepair(t, false) }
func TestShortReorgedRepairWithSnapshots(t *testing.T) { testShortReorgedRepair(t, true) }
func testShortReorgedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -421,7 +468,7 @@ func TestShortReorgedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a longer side chain, where
@ -429,6 +476,13 @@ func TestShortReorgedRepair(t *testing.T) {
// crashed. In this case we expect the canonical chain to be rolled back to the
// committed block, but the chain data itself left in the database for replaying.
func TestShortReorgedFastSyncedRepair(t *testing.T) {
testShortReorgedFastSyncedRepair(t, false)
}
func TestShortReorgedFastSyncedRepairWithSnapshots(t *testing.T) {
testShortReorgedFastSyncedRepair(t, true)
}
func testShortReorgedFastSyncedRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -460,7 +514,7 @@ func TestShortReorgedFastSyncedRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a short canonical chain and a longer side chain, where
@ -468,6 +522,13 @@ func TestShortReorgedFastSyncedRepair(t *testing.T) {
// this case we expect the chain to detect that it was fast syncing and not delete
// anything, since we can just pick up directly where we left off.
func TestShortReorgedFastSyncingRepair(t *testing.T) {
testShortReorgedFastSyncingRepair(t, false)
}
func TestShortReorgedFastSyncingRepairWithSnapshots(t *testing.T) {
testShortReorgedFastSyncingRepair(t, true)
}
func testShortReorgedFastSyncingRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -499,14 +560,17 @@ func TestShortReorgedFastSyncingRepair(t *testing.T) {
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where a recent
// block - newer than the ancient limit - was already committed to disk and then
// the process crashed. In this case we expect the chain to be rolled back to the
// committed block, with everything afterwads kept as fast sync data.
func TestLongShallowRepair(t *testing.T) {
func TestLongShallowRepair(t *testing.T) { testLongShallowRepair(t, false) }
func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) }
func testLongShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -541,14 +605,17 @@ func TestLongShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where a recent
// block - older than the ancient limit - was already committed to disk and then
// the process crashed. In this case we expect the chain to be rolled back to the
// committed block, with everything afterwads deleted.
func TestLongDeepRepair(t *testing.T) {
func TestLongDeepRepair(t *testing.T) { testLongDeepRepair(t, false) }
func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) }
func testLongDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -582,7 +649,7 @@ func TestLongDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where the fast
@ -590,6 +657,13 @@ func TestLongDeepRepair(t *testing.T) {
// which the process crashed. In this case we expect the chain to be rolled back
// to the committed block, with everything afterwads kept as fast sync data.
func TestLongFastSyncedShallowRepair(t *testing.T) {
testLongFastSyncedShallowRepair(t, false)
}
func TestLongFastSyncedShallowRepairWithSnapshots(t *testing.T) {
testLongFastSyncedShallowRepair(t, true)
}
func testLongFastSyncedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -624,14 +698,17 @@ func TestLongFastSyncedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where the fast
// sync pivot point - older than the ancient limit - was already committed, after
// which the process crashed. In this case we expect the chain to be rolled back
// to the committed block, with everything afterwads deleted.
func TestLongFastSyncedDeepRepair(t *testing.T) {
func TestLongFastSyncedDeepRepair(t *testing.T) { testLongFastSyncedDeepRepair(t, false) }
func TestLongFastSyncedDeepRepairWithSnapshots(t *testing.T) { testLongFastSyncedDeepRepair(t, true) }
func testLongFastSyncedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -665,7 +742,7 @@ func TestLongFastSyncedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where the fast
@ -674,6 +751,13 @@ func TestLongFastSyncedDeepRepair(t *testing.T) {
// syncing and not delete anything, since we can just pick up directly where we
// left off.
func TestLongFastSyncingShallowRepair(t *testing.T) {
testLongFastSyncingShallowRepair(t, false)
}
func TestLongFastSyncingShallowRepairWithSnapshots(t *testing.T) {
testLongFastSyncingShallowRepair(t, true)
}
func testLongFastSyncingShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -708,7 +792,7 @@ func TestLongFastSyncingShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks where the fast
@ -716,7 +800,10 @@ func TestLongFastSyncingShallowRepair(t *testing.T) {
// process crashed. In this case we expect the chain to detect that it was fast
// syncing and not delete anything, since we can just pick up directly where we
// left off.
func TestLongFastSyncingDeepRepair(t *testing.T) {
func TestLongFastSyncingDeepRepair(t *testing.T) { testLongFastSyncingDeepRepair(t, false) }
func TestLongFastSyncingDeepRepairWithSnapshots(t *testing.T) { testLongFastSyncingDeepRepair(t, true) }
func testLongFastSyncingDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -751,7 +838,7 @@ func TestLongFastSyncingDeepRepair(t *testing.T) {
expHeadHeader: 24,
expHeadFastBlock: 24,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -761,6 +848,13 @@ func TestLongFastSyncingDeepRepair(t *testing.T) {
// rolled back to the committed block, with everything afterwads kept as fast
// sync data; the side chain completely nuked by the freezer.
func TestLongOldForkedShallowRepair(t *testing.T) {
testLongOldForkedShallowRepair(t, false)
}
func TestLongOldForkedShallowRepairWithSnapshots(t *testing.T) {
testLongOldForkedShallowRepair(t, true)
}
func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -796,7 +890,7 @@ func TestLongOldForkedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -805,7 +899,10 @@ func TestLongOldForkedShallowRepair(t *testing.T) {
// chain is below the committed block. In this case we expect the canonical chain
// to be rolled back to the committed block, with everything afterwads deleted;
// the side chain completely nuked by the freezer.
func TestLongOldForkedDeepRepair(t *testing.T) {
func TestLongOldForkedDeepRepair(t *testing.T) { testLongOldForkedDeepRepair(t, false) }
func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) }
func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -840,7 +937,7 @@ func TestLongOldForkedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -850,6 +947,13 @@ func TestLongOldForkedDeepRepair(t *testing.T) {
// to be rolled back to the committed block, with everything afterwads kept as
// fast sync data; the side chain completely nuked by the freezer.
func TestLongOldForkedFastSyncedShallowRepair(t *testing.T) {
testLongOldForkedFastSyncedShallowRepair(t, false)
}
func TestLongOldForkedFastSyncedShallowRepairWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncedShallowRepair(t, true)
}
func testLongOldForkedFastSyncedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -885,7 +989,7 @@ func TestLongOldForkedFastSyncedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -895,6 +999,13 @@ func TestLongOldForkedFastSyncedShallowRepair(t *testing.T) {
// chain to be rolled back to the committed block, with everything afterwads deleted;
// the side chain completely nuked by the freezer.
func TestLongOldForkedFastSyncedDeepRepair(t *testing.T) {
testLongOldForkedFastSyncedDeepRepair(t, false)
}
func TestLongOldForkedFastSyncedDeepRepairWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncedDeepRepair(t, true)
}
func testLongOldForkedFastSyncedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -929,7 +1040,7 @@ func TestLongOldForkedFastSyncedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -939,6 +1050,13 @@ func TestLongOldForkedFastSyncedDeepRepair(t *testing.T) {
// that it was fast syncing and not delete anything. The side chain is completely
// nuked by the freezer.
func TestLongOldForkedFastSyncingShallowRepair(t *testing.T) {
testLongOldForkedFastSyncingShallowRepair(t, false)
}
func TestLongOldForkedFastSyncingShallowRepairWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncingShallowRepair(t, true)
}
func testLongOldForkedFastSyncingShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -974,7 +1092,7 @@ func TestLongOldForkedFastSyncingShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -984,6 +1102,13 @@ func TestLongOldForkedFastSyncingShallowRepair(t *testing.T) {
// that it was fast syncing and not delete anything. The side chain is completely
// nuked by the freezer.
func TestLongOldForkedFastSyncingDeepRepair(t *testing.T) {
testLongOldForkedFastSyncingDeepRepair(t, false)
}
func TestLongOldForkedFastSyncingDeepRepairWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncingDeepRepair(t, true)
}
func testLongOldForkedFastSyncingDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -1019,7 +1144,7 @@ func TestLongOldForkedFastSyncingDeepRepair(t *testing.T) {
expHeadHeader: 24,
expHeadFastBlock: 24,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1029,6 +1154,13 @@ func TestLongOldForkedFastSyncingDeepRepair(t *testing.T) {
// rolled back to the committed block, with everything afterwads kept as fast
// sync data; the side chain completely nuked by the freezer.
func TestLongNewerForkedShallowRepair(t *testing.T) {
testLongNewerForkedShallowRepair(t, false)
}
func TestLongNewerForkedShallowRepairWithSnapshots(t *testing.T) {
testLongNewerForkedShallowRepair(t, true)
}
func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1064,7 +1196,7 @@ func TestLongNewerForkedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1073,7 +1205,10 @@ func TestLongNewerForkedShallowRepair(t *testing.T) {
// chain is above the committed block. In this case we expect the canonical chain
// to be rolled back to the committed block, with everything afterwads deleted;
// the side chain completely nuked by the freezer.
func TestLongNewerForkedDeepRepair(t *testing.T) {
func TestLongNewerForkedDeepRepair(t *testing.T) { testLongNewerForkedDeepRepair(t, false) }
func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) }
func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1108,7 +1243,7 @@ func TestLongNewerForkedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1118,6 +1253,13 @@ func TestLongNewerForkedDeepRepair(t *testing.T) {
// to be rolled back to the committed block, with everything afterwads kept as fast
// sync data; the side chain completely nuked by the freezer.
func TestLongNewerForkedFastSyncedShallowRepair(t *testing.T) {
testLongNewerForkedFastSyncedShallowRepair(t, false)
}
func TestLongNewerForkedFastSyncedShallowRepairWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncedShallowRepair(t, true)
}
func testLongNewerForkedFastSyncedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1153,7 +1295,7 @@ func TestLongNewerForkedFastSyncedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1163,6 +1305,13 @@ func TestLongNewerForkedFastSyncedShallowRepair(t *testing.T) {
// chain to be rolled back to the committed block, with everything afterwads deleted;
// the side chain completely nuked by the freezer.
func TestLongNewerForkedFastSyncedDeepRepair(t *testing.T) {
testLongNewerForkedFastSyncedDeepRepair(t, false)
}
func TestLongNewerForkedFastSyncedDeepRepairWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncedDeepRepair(t, true)
}
func testLongNewerForkedFastSyncedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1197,7 +1346,7 @@ func TestLongNewerForkedFastSyncedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1207,6 +1356,13 @@ func TestLongNewerForkedFastSyncedDeepRepair(t *testing.T) {
// that it was fast syncing and not delete anything. The side chain is completely
// nuked by the freezer.
func TestLongNewerForkedFastSyncingShallowRepair(t *testing.T) {
testLongNewerForkedFastSyncingShallowRepair(t, false)
}
func TestLongNewerForkedFastSyncingShallowRepairWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncingShallowRepair(t, true)
}
func testLongNewerForkedFastSyncingShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1242,7 +1398,7 @@ func TestLongNewerForkedFastSyncingShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a shorter
@ -1252,6 +1408,13 @@ func TestLongNewerForkedFastSyncingShallowRepair(t *testing.T) {
// that it was fast syncing and not delete anything. The side chain is completely
// nuked by the freezer.
func TestLongNewerForkedFastSyncingDeepRepair(t *testing.T) {
testLongNewerForkedFastSyncingDeepRepair(t, false)
}
func TestLongNewerForkedFastSyncingDeepRepairWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncingDeepRepair(t, true)
}
func testLongNewerForkedFastSyncingDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1287,7 +1450,7 @@ func TestLongNewerForkedFastSyncingDeepRepair(t *testing.T) {
expHeadHeader: 24,
expHeadFastBlock: 24,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer side
@ -1295,7 +1458,10 @@ func TestLongNewerForkedFastSyncingDeepRepair(t *testing.T) {
// to disk and then the process crashed. In this case we expect the chain to be
// rolled back to the committed block, with everything afterwads kept as fast sync
// data. The side chain completely nuked by the freezer.
func TestLongReorgedShallowRepair(t *testing.T) {
func TestLongReorgedShallowRepair(t *testing.T) { testLongReorgedShallowRepair(t, false) }
func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) }
func testLongReorgedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1331,7 +1497,7 @@ func TestLongReorgedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer side
@ -1339,7 +1505,10 @@ func TestLongReorgedShallowRepair(t *testing.T) {
// to disk and then the process crashed. In this case we expect the canonical chains
// to be rolled back to the committed block, with everything afterwads deleted. The
// side chain completely nuked by the freezer.
func TestLongReorgedDeepRepair(t *testing.T) {
func TestLongReorgedDeepRepair(t *testing.T) { testLongReorgedDeepRepair(t, false) }
func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) }
func testLongReorgedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1374,7 +1543,7 @@ func TestLongReorgedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer
@ -1384,6 +1553,13 @@ func TestLongReorgedDeepRepair(t *testing.T) {
// afterwads kept as fast sync data. The side chain completely nuked by the
// freezer.
func TestLongReorgedFastSyncedShallowRepair(t *testing.T) {
testLongReorgedFastSyncedShallowRepair(t, false)
}
func TestLongReorgedFastSyncedShallowRepairWithSnapshots(t *testing.T) {
testLongReorgedFastSyncedShallowRepair(t, true)
}
func testLongReorgedFastSyncedShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1419,7 +1595,7 @@ func TestLongReorgedFastSyncedShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer
@ -1428,6 +1604,13 @@ func TestLongReorgedFastSyncedShallowRepair(t *testing.T) {
// expect the canonical chains to be rolled back to the committed block, with
// everything afterwads deleted. The side chain completely nuked by the freezer.
func TestLongReorgedFastSyncedDeepRepair(t *testing.T) {
testLongReorgedFastSyncedDeepRepair(t, false)
}
func TestLongReorgedFastSyncedDeepRepairWithSnapshots(t *testing.T) {
testLongReorgedFastSyncedDeepRepair(t, true)
}
func testLongReorgedFastSyncedDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1462,7 +1645,7 @@ func TestLongReorgedFastSyncedDeepRepair(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer
@ -1471,6 +1654,13 @@ func TestLongReorgedFastSyncedDeepRepair(t *testing.T) {
// chain to detect that it was fast syncing and not delete anything, since we
// can just pick up directly where we left off.
func TestLongReorgedFastSyncingShallowRepair(t *testing.T) {
testLongReorgedFastSyncingShallowRepair(t, false)
}
func TestLongReorgedFastSyncingShallowRepairWithSnapshots(t *testing.T) {
testLongReorgedFastSyncingShallowRepair(t, true)
}
func testLongReorgedFastSyncingShallowRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1506,7 +1696,7 @@ func TestLongReorgedFastSyncingShallowRepair(t *testing.T) {
expHeadHeader: 18,
expHeadFastBlock: 18,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a recovery for a long canonical chain with frozen blocks and a longer
@ -1515,6 +1705,13 @@ func TestLongReorgedFastSyncingShallowRepair(t *testing.T) {
// chain to detect that it was fast syncing and not delete anything, since we
// can just pick up directly where we left off.
func TestLongReorgedFastSyncingDeepRepair(t *testing.T) {
testLongReorgedFastSyncingDeepRepair(t, false)
}
func TestLongReorgedFastSyncingDeepRepairWithSnapshots(t *testing.T) {
testLongReorgedFastSyncingDeepRepair(t, true)
}
func testLongReorgedFastSyncingDeepRepair(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1550,13 +1747,13 @@ func TestLongReorgedFastSyncingDeepRepair(t *testing.T) {
expHeadHeader: 24,
expHeadFastBlock: 24,
expHeadBlock: 0,
})
}, snapshots)
}
func testRepair(t *testing.T, tt *rewindTest) {
func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
// It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
//fmt.Println(tt.dump(true))
// fmt.Println(tt.dump(true))
// Create a temporary persistent database
datadir, err := ioutil.TempDir("", "")
@ -1575,8 +1772,18 @@ func testRepair(t *testing.T, tt *rewindTest) {
var (
genesis = new(Genesis).MustCommit(db)
engine = ethash.NewFullFaker()
config = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot by default
}
)
chain, err := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if snapshots {
config.SnapshotLimit = 256
config.SnapshotWait = true
}
chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to create chain: %v", err)
}
@ -1599,6 +1806,11 @@ func testRepair(t *testing.T, tt *rewindTest) {
}
if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
}
}
if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)

324
core/blockchain_sethead_test.go
View file

@ -26,6 +26,7 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
@ -150,7 +151,10 @@ func (tt *rewindTest) dump(crash bool) string {
// chain to be rolled back to the committed block. Everything above the sethead
// point should be deleted. In between the committed block and the requested head
// the data can remain as "fast sync" data to avoid redownloading it.
func TestShortSetHead(t *testing.T) {
func TestShortSetHead(t *testing.T) { testShortSetHead(t, false) }
func TestShortSetHeadWithSnapshots(t *testing.T) { testShortSetHead(t, true) }
func testShortSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -181,7 +185,7 @@ func TestShortSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain where the fast sync pivot point was
@ -190,7 +194,10 @@ func TestShortSetHead(t *testing.T) {
// Everything above the sethead point should be deleted. In between the committed
// block and the requested head the data can remain as "fast sync" data to avoid
// redownloading it.
func TestShortFastSyncedSetHead(t *testing.T) {
func TestShortFastSyncedSetHead(t *testing.T) { testShortFastSyncedSetHead(t, false) }
func TestShortFastSyncedSetHeadWithSnapshots(t *testing.T) { testShortFastSyncedSetHead(t, true) }
func testShortFastSyncedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -221,7 +228,7 @@ func TestShortFastSyncedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain where the fast sync pivot point was
@ -229,7 +236,10 @@ func TestShortFastSyncedSetHead(t *testing.T) {
// detect that it was fast syncing and delete everything from the new head, since
// we can just pick up fast syncing from there. The head full block should be set
// to the genesis.
func TestShortFastSyncingSetHead(t *testing.T) {
func TestShortFastSyncingSetHead(t *testing.T) { testShortFastSyncingSetHead(t, false) }
func TestShortFastSyncingSetHeadWithSnapshots(t *testing.T) { testShortFastSyncingSetHead(t, true) }
func testShortFastSyncingSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
@ -260,7 +270,7 @@ func TestShortFastSyncingSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where a
@ -270,7 +280,10 @@ func TestShortFastSyncingSetHead(t *testing.T) {
// above the sethead point should be deleted. In between the committed block and
// the requested head the data can remain as "fast sync" data to avoid redownloading
// it. The side chain should be left alone as it was shorter.
func TestShortOldForkedSetHead(t *testing.T) {
func TestShortOldForkedSetHead(t *testing.T) { testShortOldForkedSetHead(t, false) }
func TestShortOldForkedSetHeadWithSnapshots(t *testing.T) { testShortOldForkedSetHead(t, true) }
func testShortOldForkedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -303,7 +316,7 @@ func TestShortOldForkedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where
@ -314,6 +327,13 @@ func TestShortOldForkedSetHead(t *testing.T) {
// committed block and the requested head the data can remain as "fast sync" data
// to avoid redownloading it. The side chain should be left alone as it was shorter.
func TestShortOldForkedFastSyncedSetHead(t *testing.T) {
testShortOldForkedFastSyncedSetHead(t, false)
}
func TestShortOldForkedFastSyncedSetHeadWithSnapshots(t *testing.T) {
testShortOldForkedFastSyncedSetHead(t, true)
}
func testShortOldForkedFastSyncedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -346,7 +366,7 @@ func TestShortOldForkedFastSyncedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where
@ -356,6 +376,13 @@ func TestShortOldForkedFastSyncedSetHead(t *testing.T) {
// head, since we can just pick up fast syncing from there. The head full block
// should be set to the genesis.
func TestShortOldForkedFastSyncingSetHead(t *testing.T) {
testShortOldForkedFastSyncingSetHead(t, false)
}
func TestShortOldForkedFastSyncingSetHeadWithSnapshots(t *testing.T) {
testShortOldForkedFastSyncingSetHead(t, true)
}
func testShortOldForkedFastSyncingSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3
@ -388,7 +415,7 @@ func TestShortOldForkedFastSyncingSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where a
@ -402,7 +429,10 @@ func TestShortOldForkedFastSyncingSetHead(t *testing.T) {
// The side chain could be left to be if the fork point was before the new head
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortNewlyForkedSetHead(t *testing.T) {
func TestShortNewlyForkedSetHead(t *testing.T) { testShortNewlyForkedSetHead(t, false) }
func TestShortNewlyForkedSetHeadWithSnapshots(t *testing.T) { testShortNewlyForkedSetHead(t, true) }
func testShortNewlyForkedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8
@ -435,7 +465,7 @@ func TestShortNewlyForkedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where
@ -449,6 +479,13 @@ func TestShortNewlyForkedSetHead(t *testing.T) {
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortNewlyForkedFastSyncedSetHead(t *testing.T) {
testShortNewlyForkedFastSyncedSetHead(t, false)
}
func TestShortNewlyForkedFastSyncedSetHeadWithSnapshots(t *testing.T) {
testShortNewlyForkedFastSyncedSetHead(t, true)
}
func testShortNewlyForkedFastSyncedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8
@ -481,7 +518,7 @@ func TestShortNewlyForkedFastSyncedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a shorter side chain, where
@ -495,6 +532,13 @@ func TestShortNewlyForkedFastSyncedSetHead(t *testing.T) {
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortNewlyForkedFastSyncingSetHead(t *testing.T) {
testShortNewlyForkedFastSyncingSetHead(t, false)
}
func TestShortNewlyForkedFastSyncingSetHeadWithSnapshots(t *testing.T) {
testShortNewlyForkedFastSyncingSetHead(t, true)
}
func testShortNewlyForkedFastSyncingSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8
@ -527,7 +571,7 @@ func TestShortNewlyForkedFastSyncingSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a longer side chain, where a
@ -540,7 +584,10 @@ func TestShortNewlyForkedFastSyncingSetHead(t *testing.T) {
// The side chain could be left to be if the fork point was before the new head
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortReorgedSetHead(t *testing.T) {
func TestShortReorgedSetHead(t *testing.T) { testShortReorgedSetHead(t, false) }
func TestShortReorgedSetHeadWithSnapshots(t *testing.T) { testShortReorgedSetHead(t, true) }
func testShortReorgedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -573,7 +620,7 @@ func TestShortReorgedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a longer side chain, where
@ -588,6 +635,13 @@ func TestShortReorgedSetHead(t *testing.T) {
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortReorgedFastSyncedSetHead(t *testing.T) {
testShortReorgedFastSyncedSetHead(t, false)
}
func TestShortReorgedFastSyncedSetHeadWithSnapshots(t *testing.T) {
testShortReorgedFastSyncedSetHead(t, true)
}
func testShortReorgedFastSyncedSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -620,7 +674,7 @@ func TestShortReorgedFastSyncedSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a short canonical chain and a longer side chain, where
@ -633,6 +687,13 @@ func TestShortReorgedFastSyncedSetHead(t *testing.T) {
// we are deleting to, but it would be exceedingly hard to detect that case and
// properly handle it, so we'll trade extra work in exchange for simpler code.
func TestShortReorgedFastSyncingSetHead(t *testing.T) {
testShortReorgedFastSyncingSetHead(t, false)
}
func TestShortReorgedFastSyncingSetHeadWithSnapshots(t *testing.T) {
testShortReorgedFastSyncingSetHead(t, true)
}
func testShortReorgedFastSyncingSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10
@ -665,7 +726,7 @@ func TestShortReorgedFastSyncingSetHead(t *testing.T) {
expHeadHeader: 7,
expHeadFastBlock: 7,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where a recent
@ -674,7 +735,10 @@ func TestShortReorgedFastSyncingSetHead(t *testing.T) {
// to the committed block. Everything above the sethead point should be deleted.
// In between the committed block and the requested head the data can remain as
// "fast sync" data to avoid redownloading it.
func TestLongShallowSetHead(t *testing.T) {
func TestLongShallowSetHead(t *testing.T) { testLongShallowSetHead(t, false) }
func TestLongShallowSetHeadWithSnapshots(t *testing.T) { testLongShallowSetHead(t, true) }
func testLongShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -710,7 +774,7 @@ func TestLongShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where a recent
@ -718,7 +782,10 @@ func TestLongShallowSetHead(t *testing.T) {
// sethead was called. In this case we expect the full chain to be rolled back
// to the committed block. Since the ancient limit was underflown, everything
// needs to be deleted onwards to avoid creating a gap.
func TestLongDeepSetHead(t *testing.T) {
func TestLongDeepSetHead(t *testing.T) { testLongDeepSetHead(t, false) }
func TestLongDeepSetHeadWithSnapshots(t *testing.T) { testLongDeepSetHead(t, true) }
func testLongDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -753,7 +820,7 @@ func TestLongDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where the fast
@ -763,6 +830,13 @@ func TestLongDeepSetHead(t *testing.T) {
// deleted. In between the committed block and the requested head the data can
// remain as "fast sync" data to avoid redownloading it.
func TestLongFastSyncedShallowSetHead(t *testing.T) {
testLongFastSyncedShallowSetHead(t, false)
}
func TestLongFastSyncedShallowSetHeadWithSnapshots(t *testing.T) {
testLongFastSyncedShallowSetHead(t, true)
}
func testLongFastSyncedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -798,7 +872,7 @@ func TestLongFastSyncedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where the fast
@ -806,7 +880,10 @@ func TestLongFastSyncedShallowSetHead(t *testing.T) {
// which sethead was called. In this case we expect the full chain to be rolled
// back to the committed block. Since the ancient limit was underflown, everything
// needs to be deleted onwards to avoid creating a gap.
func TestLongFastSyncedDeepSetHead(t *testing.T) {
func TestLongFastSyncedDeepSetHead(t *testing.T) { testLongFastSyncedDeepSetHead(t, false) }
func TestLongFastSyncedDeepSetHeadWithSnapshots(t *testing.T) { testLongFastSyncedDeepSetHead(t, true) }
func testLongFastSyncedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -841,7 +918,7 @@ func TestLongFastSyncedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where the fast
@ -850,6 +927,13 @@ func TestLongFastSyncedDeepSetHead(t *testing.T) {
// syncing and delete everything from the new head, since we can just pick up fast
// syncing from there.
func TestLongFastSyncingShallowSetHead(t *testing.T) {
testLongFastSyncingShallowSetHead(t, false)
}
func TestLongFastSyncingShallowSetHeadWithSnapshots(t *testing.T) {
testLongFastSyncingShallowSetHead(t, true)
}
func testLongFastSyncingShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
//
@ -885,7 +969,7 @@ func TestLongFastSyncingShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks where the fast
@ -894,6 +978,13 @@ func TestLongFastSyncingShallowSetHead(t *testing.T) {
// syncing and delete everything from the new head, since we can just pick up fast
// syncing from there.
func TestLongFastSyncingDeepSetHead(t *testing.T) {
testLongFastSyncingDeepSetHead(t, false)
}
func TestLongFastSyncingDeepSetHeadWithSnapshots(t *testing.T) {
testLongFastSyncingDeepSetHead(t, true)
}
func testLongFastSyncingDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
//
@ -928,7 +1019,7 @@ func TestLongFastSyncingDeepSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter side
@ -939,6 +1030,13 @@ func TestLongFastSyncingDeepSetHead(t *testing.T) {
// can remain as "fast sync" data to avoid redownloading it. The side chain is nuked
// by the freezer.
func TestLongOldForkedShallowSetHead(t *testing.T) {
testLongOldForkedShallowSetHead(t, false)
}
func TestLongOldForkedShallowSetHeadWithSnapshots(t *testing.T) {
testLongOldForkedShallowSetHead(t, true)
}
func testLongOldForkedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -975,7 +1073,7 @@ func TestLongOldForkedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter side
@ -984,7 +1082,10 @@ func TestLongOldForkedShallowSetHead(t *testing.T) {
// chain to be rolled back to the committed block. Since the ancient limit was
// underflown, everything needs to be deleted onwards to avoid creating a gap. The
// side chain is nuked by the freezer.
func TestLongOldForkedDeepSetHead(t *testing.T) {
func TestLongOldForkedDeepSetHead(t *testing.T) { testLongOldForkedDeepSetHead(t, false) }
func TestLongOldForkedDeepSetHeadWithSnapshots(t *testing.T) { testLongOldForkedDeepSetHead(t, true) }
func testLongOldForkedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -1020,7 +1121,7 @@ func TestLongOldForkedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1032,6 +1133,13 @@ func TestLongOldForkedDeepSetHead(t *testing.T) {
// requested head the data can remain as "fast sync" data to avoid redownloading
// it. The side chain is nuked by the freezer.
func TestLongOldForkedFastSyncedShallowSetHead(t *testing.T) {
testLongOldForkedFastSyncedShallowSetHead(t, false)
}
func TestLongOldForkedFastSyncedShallowSetHeadWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncedShallowSetHead(t, true)
}
func testLongOldForkedFastSyncedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -1068,7 +1176,7 @@ func TestLongOldForkedFastSyncedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1079,6 +1187,13 @@ func TestLongOldForkedFastSyncedShallowSetHead(t *testing.T) {
// underflown, everything needs to be deleted onwards to avoid creating a gap. The
// side chain is nuked by the freezer.
func TestLongOldForkedFastSyncedDeepSetHead(t *testing.T) {
testLongOldForkedFastSyncedDeepSetHead(t, false)
}
func TestLongOldForkedFastSyncedDeepSetHeadWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncedDeepSetHead(t, true)
}
func testLongOldForkedFastSyncedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -1114,7 +1229,7 @@ func TestLongOldForkedFastSyncedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1125,6 +1240,13 @@ func TestLongOldForkedFastSyncedDeepSetHead(t *testing.T) {
// just pick up fast syncing from there. The side chain is completely nuked by the
// freezer.
func TestLongOldForkedFastSyncingShallowSetHead(t *testing.T) {
testLongOldForkedFastSyncingShallowSetHead(t, false)
}
func TestLongOldForkedFastSyncingShallowSetHeadWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncingShallowSetHead(t, true)
}
func testLongOldForkedFastSyncingShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3
@ -1161,7 +1283,7 @@ func TestLongOldForkedFastSyncingShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1172,6 +1294,13 @@ func TestLongOldForkedFastSyncingShallowSetHead(t *testing.T) {
// just pick up fast syncing from there. The side chain is completely nuked by the
// freezer.
func TestLongOldForkedFastSyncingDeepSetHead(t *testing.T) {
testLongOldForkedFastSyncingDeepSetHead(t, false)
}
func TestLongOldForkedFastSyncingDeepSetHeadWithSnapshots(t *testing.T) {
testLongOldForkedFastSyncingDeepSetHead(t, true)
}
func testLongOldForkedFastSyncingDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3
@ -1207,7 +1336,7 @@ func TestLongOldForkedFastSyncingDeepSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1216,6 +1345,13 @@ func TestLongOldForkedFastSyncingDeepSetHead(t *testing.T) {
// chain is above the committed block. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongShallowSetHead.
func TestLongNewerForkedShallowSetHead(t *testing.T) {
testLongNewerForkedShallowSetHead(t, false)
}
func TestLongNewerForkedShallowSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedShallowSetHead(t, true)
}
func testLongNewerForkedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1252,7 +1388,7 @@ func TestLongNewerForkedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1261,6 +1397,13 @@ func TestLongNewerForkedShallowSetHead(t *testing.T) {
// chain is above the committed block. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongDeepSetHead.
func TestLongNewerForkedDeepSetHead(t *testing.T) {
testLongNewerForkedDeepSetHead(t, false)
}
func TestLongNewerForkedDeepSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedDeepSetHead(t, true)
}
func testLongNewerForkedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1296,7 +1439,7 @@ func TestLongNewerForkedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1305,6 +1448,13 @@ func TestLongNewerForkedDeepSetHead(t *testing.T) {
// the side chain is above the committed block. In this case the freezer will delete
// the sidechain since it's dangling, reverting to TestLongFastSyncedShallowSetHead.
func TestLongNewerForkedFastSyncedShallowSetHead(t *testing.T) {
testLongNewerForkedFastSyncedShallowSetHead(t, false)
}
func TestLongNewerForkedFastSyncedShallowSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncedShallowSetHead(t, true)
}
func testLongNewerForkedFastSyncedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1341,7 +1491,7 @@ func TestLongNewerForkedFastSyncedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1350,6 +1500,13 @@ func TestLongNewerForkedFastSyncedShallowSetHead(t *testing.T) {
// the side chain is above the committed block. In this case the freezer will delete
// the sidechain since it's dangling, reverting to TestLongFastSyncedDeepSetHead.
func TestLongNewerForkedFastSyncedDeepSetHead(t *testing.T) {
testLongNewerForkedFastSyncedDeepSetHead(t, false)
}
func TestLongNewerForkedFastSyncedDeepSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncedDeepSetHead(t, true)
}
func testLongNewerForkedFastSyncedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1385,7 +1542,7 @@ func TestLongNewerForkedFastSyncedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1394,6 +1551,13 @@ func TestLongNewerForkedFastSyncedDeepSetHead(t *testing.T) {
// chain is above the committed block. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongFastSyncinghallowSetHead.
func TestLongNewerForkedFastSyncingShallowSetHead(t *testing.T) {
testLongNewerForkedFastSyncingShallowSetHead(t, false)
}
func TestLongNewerForkedFastSyncingShallowSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncingShallowSetHead(t, true)
}
func testLongNewerForkedFastSyncingShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1430,7 +1594,7 @@ func TestLongNewerForkedFastSyncingShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a shorter
@ -1439,6 +1603,13 @@ func TestLongNewerForkedFastSyncingShallowSetHead(t *testing.T) {
// chain is above the committed block. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongFastSyncingDeepSetHead.
func TestLongNewerForkedFastSyncingDeepSetHead(t *testing.T) {
testLongNewerForkedFastSyncingDeepSetHead(t, false)
}
func TestLongNewerForkedFastSyncingDeepSetHeadWithSnapshots(t *testing.T) {
testLongNewerForkedFastSyncingDeepSetHead(t, true)
}
func testLongNewerForkedFastSyncingDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12
@ -1474,14 +1645,17 @@ func TestLongNewerForkedFastSyncingDeepSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer side
// chain, where a recent block - newer than the ancient limit - was already committed
// to disk and then sethead was called. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongShallowSetHead.
func TestLongReorgedShallowSetHead(t *testing.T) {
func TestLongReorgedShallowSetHead(t *testing.T) { testLongReorgedShallowSetHead(t, false) }
func TestLongReorgedShallowSetHeadWithSnapshots(t *testing.T) { testLongReorgedShallowSetHead(t, true) }
func testLongReorgedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1518,14 +1692,17 @@ func TestLongReorgedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer side
// chain, where a recent block - older than the ancient limit - was already committed
// to disk and then sethead was called. In this case the freezer will delete the
// sidechain since it's dangling, reverting to TestLongDeepSetHead.
func TestLongReorgedDeepSetHead(t *testing.T) {
func TestLongReorgedDeepSetHead(t *testing.T) { testLongReorgedDeepSetHead(t, false) }
func TestLongReorgedDeepSetHeadWithSnapshots(t *testing.T) { testLongReorgedDeepSetHead(t, true) }
func testLongReorgedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1561,7 +1738,7 @@ func TestLongReorgedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer
@ -1570,6 +1747,13 @@ func TestLongReorgedDeepSetHead(t *testing.T) {
// freezer will delete the sidechain since it's dangling, reverting to
// TestLongFastSyncedShallowSetHead.
func TestLongReorgedFastSyncedShallowSetHead(t *testing.T) {
testLongReorgedFastSyncedShallowSetHead(t, false)
}
func TestLongReorgedFastSyncedShallowSetHeadWithSnapshots(t *testing.T) {
testLongReorgedFastSyncedShallowSetHead(t, true)
}
func testLongReorgedFastSyncedShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1606,7 +1790,7 @@ func TestLongReorgedFastSyncedShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer
@ -1615,6 +1799,13 @@ func TestLongReorgedFastSyncedShallowSetHead(t *testing.T) {
// freezer will delete the sidechain since it's dangling, reverting to
// TestLongFastSyncedDeepSetHead.
func TestLongReorgedFastSyncedDeepSetHead(t *testing.T) {
testLongReorgedFastSyncedDeepSetHead(t, false)
}
func TestLongReorgedFastSyncedDeepSetHeadWithSnapshots(t *testing.T) {
testLongReorgedFastSyncedDeepSetHead(t, true)
}
func testLongReorgedFastSyncedDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1650,7 +1841,7 @@ func TestLongReorgedFastSyncedDeepSetHead(t *testing.T) {
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer
@ -1660,6 +1851,13 @@ func TestLongReorgedFastSyncedDeepSetHead(t *testing.T) {
// head, since we can just pick up fast syncing from there. The side chain is
// completely nuked by the freezer.
func TestLongReorgedFastSyncingShallowSetHead(t *testing.T) {
testLongReorgedFastSyncingShallowSetHead(t, false)
}
func TestLongReorgedFastSyncingShallowSetHeadWithSnapshots(t *testing.T) {
testLongReorgedFastSyncingShallowSetHead(t, true)
}
func testLongReorgedFastSyncingShallowSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1696,7 +1894,7 @@ func TestLongReorgedFastSyncingShallowSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
// Tests a sethead for a long canonical chain with frozen blocks and a longer
@ -1706,6 +1904,13 @@ func TestLongReorgedFastSyncingShallowSetHead(t *testing.T) {
// head, since we can just pick up fast syncing from there. The side chain is
// completely nuked by the freezer.
func TestLongReorgedFastSyncingDeepSetHead(t *testing.T) {
testLongReorgedFastSyncingDeepSetHead(t, false)
}
func TestLongReorgedFastSyncingDeepSetHeadWithSnapshots(t *testing.T) {
testLongReorgedFastSyncingDeepSetHead(t, true)
}
func testLongReorgedFastSyncingDeepSetHead(t *testing.T, snapshots bool) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10->C11->C12->C13->C14->C15->C16->C17->C18->C19->C20->C21->C22->C23->C24 (HEAD)
// └->S1->S2->S3->S4->S5->S6->S7->S8->S9->S10->S11->S12->S13->S14->S15->S16->S17->S18->S19->S20->S21->S22->S23->S24->S25->S26
@ -1741,13 +1946,13 @@ func TestLongReorgedFastSyncingDeepSetHead(t *testing.T) {
expHeadHeader: 6,
expHeadFastBlock: 6,
expHeadBlock: 0,
})
}, snapshots)
}
func testSetHead(t *testing.T, tt *rewindTest) {
func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
// It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
//fmt.Println(tt.dump(false))
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(false))
// Create a temporary persistent database
datadir, err := ioutil.TempDir("", "")
@ -1766,8 +1971,18 @@ func testSetHead(t *testing.T, tt *rewindTest) {
var (
genesis = new(Genesis).MustCommit(db)
engine = ethash.NewFullFaker()
config = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0, // Disable snapshot
}
)
chain, err := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if snapshots {
config.SnapshotLimit = 256
config.SnapshotWait = true
}
chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to create chain: %v", err)
}
@ -1790,6 +2005,11 @@ func testSetHead(t *testing.T, tt *rewindTest) {
}
if tt.commitBlock > 0 {
chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
if snapshots {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
}
}
if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)

797
core/blockchain_snapshot_test.go Normal file
View file

@ -0,0 +1,797 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Tests that abnormal program termination (i.e.crash) and restart can recovery
// the snapshot properly if the snapshot is enabled.
package core
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"time"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
)
// snapshotTest is a test case for snapshot recovery. It can be used for
// simulating these scenarios:
// (i) Geth restarts normally with valid legacy snapshot
// (ii) Geth restarts normally with valid new-format snapshot
// (iii) Geth restarts after the crash, with broken legacy snapshot
// (iv) Geth restarts after the crash, with broken new-format snapshot
// (v) Geth restarts normally, but it's requested to be rewound to a lower point via SetHead
// (vi) Geth restarts normally with a stale snapshot
type snapshotTest struct {
legacy bool // Flag whether the loaded snapshot is in legacy format
crash bool // Flag whether the Geth restarts from the previous crash
restartCrash int // Number of blocks to insert after the normal stop, then the crash happens
gapped int // Number of blocks to insert without enabling snapshot
setHead uint64 // Block number to set head back to
chainBlocks int // Number of blocks to generate for the canonical chain
snapshotBlock uint64 // Block number of the relevant snapshot disk layer
commitBlock uint64 // Block number for which to commit the state to disk
expCanonicalBlocks int // Number of canonical blocks expected to remain in the database (excl. genesis)
expHeadHeader uint64 // Block number of the expected head header
expHeadFastBlock uint64 // Block number of the expected head fast sync block
expHeadBlock uint64 // Block number of the expected head full block
expSnapshotBottom uint64 // The block height corresponding to the snapshot disk layer
}
func (tt *snapshotTest) dump() string {
buffer := new(strings.Builder)
fmt.Fprint(buffer, "Chain:\n G")
for i := 0; i < tt.chainBlocks; i++ {
fmt.Fprintf(buffer, "->C%d", i+1)
}
fmt.Fprint(buffer, " (HEAD)\n\n")
fmt.Fprintf(buffer, "Commit: G")
if tt.commitBlock > 0 {
fmt.Fprintf(buffer, ", C%d", tt.commitBlock)
}
fmt.Fprint(buffer, "\n")
fmt.Fprintf(buffer, "Snapshot: G")
if tt.snapshotBlock > 0 {
fmt.Fprintf(buffer, ", C%d", tt.snapshotBlock)
}
fmt.Fprint(buffer, "\n")
if tt.crash {
fmt.Fprintf(buffer, "\nCRASH\n\n")
} else {
fmt.Fprintf(buffer, "\nSetHead(%d)\n\n", tt.setHead)
}
fmt.Fprintf(buffer, "------------------------------\n\n")
fmt.Fprint(buffer, "Expected in leveldb:\n G")
for i := 0; i < tt.expCanonicalBlocks; i++ {
fmt.Fprintf(buffer, "->C%d", i+1)
}
fmt.Fprintf(buffer, "\n\n")
fmt.Fprintf(buffer, "Expected head header : C%d\n", tt.expHeadHeader)
fmt.Fprintf(buffer, "Expected head fast block: C%d\n", tt.expHeadFastBlock)
if tt.expHeadBlock == 0 {
fmt.Fprintf(buffer, "Expected head block : G\n")
} else {
fmt.Fprintf(buffer, "Expected head block : C%d\n", tt.expHeadBlock)
}
if tt.expSnapshotBottom == 0 {
fmt.Fprintf(buffer, "Expected snapshot disk : G\n")
} else {
fmt.Fprintf(buffer, "Expected snapshot disk : C%d\n", tt.expSnapshotBottom)
}
return buffer.String()
}
// Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
// journal will be persisted correctly. In this case no snapshot recovery is
// required.
func TestRestartWithNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(0)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : C8
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: false,
crash: false,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 8,
expSnapshotBottom: 0, // Initial disk layer built from genesis
})
}
// Tests a Geth restart with valid but "legacy" snapshot. Before the shutdown,
// all snapshot journal will be persisted correctly. In this case no snapshot
// recovery is required.
func TestRestartWithLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(0)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : C8
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: true,
crash: false,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 8,
expSnapshotBottom: 0, // Initial disk layer built from genesis
})
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the
// chain head should be rewound to the point with available state. And also the
// new head should must be lower than disk layer. But there is no committed point
// so the chain should be rewound to genesis and the disk layer should be left
// for recovery.
func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
testSnapshot(t, &snapshotTest{
legacy: false,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
})
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case the
// chain head should be rewound to the point with available state. And also the
// new head should must be lower than disk layer. But there is only a low committed
// point so the chain should be rewound to committed point and the disk layer
// should be left for recovery.
func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G, C2
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : C2
// Expected snapshot disk : C4
testSnapshot(t, &snapshotTest{
legacy: false,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 2,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 2,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
})
}
// Tests a Geth was crashed and restarts with a broken snapshot. In this case
// the chain head should be rewound to the point with available state. And also
// the new head should must be lower than disk layer. But there is only a high
// committed point so the chain should be rewound to genesis and the disk layer
// should be left for recovery.
func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G, C6
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
testSnapshot(t, &snapshotTest{
legacy: false,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 6,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 4, // Last committed disk layer, wait recovery
})
}
// Tests a Geth was crashed and restarts with a broken and "legacy format"
// snapshot. In this case the entire legacy snapshot should be discared
// and rebuild from the new chain head. The new head here refers to the
// genesis because there is no committed point.
func TestNoCommitCrashWithLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: true,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 0,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis)
})
}
// Tests a Geth was crashed and restarts with a broken and "legacy format"
// snapshot. In this case the entire legacy snapshot should be discared
// and rebuild from the new chain head. The new head here refers to the
// block-2 because it's committed into the disk.
func TestLowCommitCrashWithLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G, C2
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : C2
// Expected snapshot disk : C2
testSnapshot(t, &snapshotTest{
legacy: true,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 2,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 2,
expSnapshotBottom: 2, // Rebuilt snapshot from the latest HEAD
})
}
// Tests a Geth was crashed and restarts with a broken and "legacy format"
// snapshot. In this case the entire legacy snapshot should be discared
// and rebuild from the new chain head.
//
// The new head here refers to the the genesis, the reason is:
// - the state of block-6 is committed into the disk
// - the legacy disk layer of block-4 is committed into the disk
// - the head is rewound the genesis in order to find an available
// state lower than disk layer
func TestHighCommitCrashWithLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G, C6
// Snapshot: G, C4
//
// CRASH
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: true,
crash: true,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 4,
commitBlock: 6,
expCanonicalBlocks: 8,
expHeadHeader: 8,
expHeadFastBlock: 8,
expHeadBlock: 0,
expSnapshotBottom: 0, // Rebuilt snapshot from the latest HEAD(genesis)
})
}
// Tests a Geth was running with snapshot enabled. Then restarts without
// enabling snapshot and after that re-enable the snapshot again. In this
// case the snapshot should be rebuilt with latest chain head.
func TestGappedNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(0)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
//
// Expected head header : C10
// Expected head fast block: C10
// Expected head block : C10
// Expected snapshot disk : C10
testSnapshot(t, &snapshotTest{
legacy: false,
crash: false,
gapped: 2,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD
})
}
// Tests a Geth was running with leagcy snapshot enabled. Then restarts
// without enabling snapshot and after that re-enable the snapshot again.
// In this case the snapshot should be rebuilt with latest chain head.
func TestGappedLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(0)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
//
// Expected head header : C10
// Expected head fast block: C10
// Expected head block : C10
// Expected snapshot disk : C10
testSnapshot(t, &snapshotTest{
legacy: true,
crash: false,
gapped: 2,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 10,
expSnapshotBottom: 10, // Rebuilt snapshot from the latest HEAD
})
}
// Tests the Geth was running with snapshot enabled and resetHead is applied.
// In this case the head is rewound to the target(with state available). After
// that the chain is restarted and the original disk layer is kept.
func TestSetHeadWithNewSnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(4)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4
//
// Expected head header : C4
// Expected head fast block: C4
// Expected head block : C4
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: false,
crash: false,
gapped: 0,
setHead: 4,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 4,
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
expSnapshotBottom: 0, // The initial disk layer is built from the genesis
})
}
// Tests the Geth was running with snapshot(legacy-format) enabled and resetHead
// is applied. In this case the head is rewound to the target(with state available).
// After that the chain is restarted and the original disk layer is kept.
func TestSetHeadWithLegacySnapshot(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(4)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4
//
// Expected head header : C4
// Expected head fast block: C4
// Expected head block : C4
// Expected snapshot disk : G
testSnapshot(t, &snapshotTest{
legacy: true,
crash: false,
gapped: 0,
setHead: 4,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 4,
expHeadHeader: 4,
expHeadFastBlock: 4,
expHeadBlock: 4,
expSnapshotBottom: 0, // The initial disk layer is built from the genesis
})
}
// Tests the Geth was running with snapshot(legacy-format) enabled and upgrades
// the disk layer journal(journal generator) to latest format. After that the Geth
// is restarted from a crash. In this case Geth will find the new-format disk layer
// journal but with legacy-format diff journal(the new-format is never committed),
// and the invalid diff journal is expected to be dropped.
func TestRecoverSnapshotFromCrashWithLegacyDiffJournal(t *testing.T) {
// Chain:
// G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
//
// Commit: G
// Snapshot: G
//
// SetHead(0)
//
// ------------------------------
//
// Expected in leveldb:
// G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
//
// Expected head header : C10
// Expected head fast block: C10
// Expected head block : C8
// Expected snapshot disk : C10
testSnapshot(t, &snapshotTest{
legacy: true,
crash: false,
restartCrash: 2,
gapped: 0,
setHead: 0,
chainBlocks: 8,
snapshotBlock: 0,
commitBlock: 0,
expCanonicalBlocks: 10,
expHeadHeader: 10,
expHeadFastBlock: 10,
expHeadBlock: 8, // The persisted state in the first running
expSnapshotBottom: 10, // The persisted disk layer in the second running
})
}
func testSnapshot(t *testing.T, tt *snapshotTest) {
// It's hard to follow the test case, visualize the input
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump())
// Create a temporary persistent database
datadir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Failed to create temporary datadir: %v", err)
}
os.RemoveAll(datadir)
db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
}
defer db.Close() // Might double close, should be fine
// Initialize a fresh chain
var (
genesis = new(Genesis).MustCommit(db)
engine = ethash.NewFullFaker()
gendb = rawdb.NewMemoryDatabase()
// Snapshot is enabled, the first snapshot is created from the Genesis.
// The snapshot memory allowance is 256MB, it means no snapshot flush
// will happen during the block insertion.
cacheConfig = defaultCacheConfig
)
chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to create chain: %v", err)
}
blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, tt.chainBlocks, func(i int, b *BlockGen) {})
// Insert the blocks with configured settings.
var breakpoints []uint64
if tt.commitBlock > tt.snapshotBlock {
breakpoints = append(breakpoints, tt.snapshotBlock, tt.commitBlock)
} else {
breakpoints = append(breakpoints, tt.commitBlock, tt.snapshotBlock)
}
var startPoint uint64
for _, point := range breakpoints {
if _, err := chain.InsertChain(blocks[startPoint:point]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
startPoint = point
if tt.commitBlock > 0 && tt.commitBlock == point {
chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil)
}
if tt.snapshotBlock > 0 && tt.snapshotBlock == point {
if tt.legacy {
// Here we commit the snapshot disk root to simulate
// committing the legacy snapshot.
rawdb.WriteSnapshotRoot(db, blocks[point-1].Root())
} else {
chain.snaps.Cap(blocks[point-1].Root(), 0)
diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root()
if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) {
t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot)
}
}
}
}
if _, err := chain.InsertChain(blocks[startPoint:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
// Set the flag for writing legacy journal if necessary
if tt.legacy {
chain.writeLegacyJournal = true
}
// Pull the plug on the database, simulating a hard crash
if tt.crash {
db.Close()
// Start a new blockchain back up and see where the repair leads us
db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
}
defer db.Close()
// The interesting thing is: instead of start the blockchain after
// the crash, we do restart twice here: one after the crash and one
// after the normal stop. It's used to ensure the broken snapshot
// can be detected all the time.
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
chain.Stop()
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer chain.Stop()
} else if tt.gapped > 0 {
// Insert blocks without enabling snapshot if gapping is required.
chain.Stop()
gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, gendb, tt.gapped, func(i int, b *BlockGen) {})
// Insert a few more blocks without enabling snapshot
var cacheConfig = &CacheConfig{
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
SnapshotLimit: 0,
}
chain, err = NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
chain.InsertChain(gappedBlocks)
chain.Stop()
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer chain.Stop()
} else if tt.setHead != 0 {
// Rewind the chain if setHead operation is required.
chain.SetHead(tt.setHead)
chain.Stop()
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer chain.Stop()
} else if tt.restartCrash != 0 {
// Firstly, stop the chain properly, with all snapshot journal
// and state committed.
chain.Stop()
// Restart chain, forcibly flush the disk layer journal with new format
newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, gendb, tt.restartCrash, func(i int, b *BlockGen) {})
chain, err = NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
chain.InsertChain(newBlocks)
chain.Snapshot().Cap(newBlocks[len(newBlocks)-1].Root(), 0)
// Simulate the blockchain crash
// Don't call chain.Stop here, so that no snapshot
// journal and latest state will be committed
// Restart the chain after the crash
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer chain.Stop()
} else {
chain.Stop()
// Restart the chain normally
chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("Failed to recreate chain: %v", err)
}
defer chain.Stop()
}
// Iterate over all the remaining blocks and ensure there are no gaps
verifyNoGaps(t, chain, true, blocks)
verifyCutoff(t, chain, true, blocks, tt.expCanonicalBlocks)
if head := chain.CurrentHeader(); head.Number.Uint64() != tt.expHeadHeader {
t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadHeader)
}
if head := chain.CurrentFastBlock(); head.NumberU64() != tt.expHeadFastBlock {
t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadFastBlock)
}
if head := chain.CurrentBlock(); head.NumberU64() != tt.expHeadBlock {
t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadBlock)
}
// Check the disk layer, ensure they are matched
block := chain.GetBlockByNumber(tt.expSnapshotBottom)
if block == nil {
t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", tt.expSnapshotBottom)
} else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
}
}

11
core/genesis.go
View file

@ -243,8 +243,8 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
return params.RinkebyChainConfig
case ghash == params.GoerliGenesisHash:
return params.GoerliChainConfig
case ghash == params.YoloV1GenesisHash:
return params.YoloV1ChainConfig
case ghash == params.YoloV2GenesisHash:
return params.YoloV2ChainConfig
default:
return params.AllEthashProtocolChanges
}
@ -380,10 +380,11 @@ func DefaultGoerliGenesisBlock() *Genesis {
}
}
func DefaultYoloV1GenesisBlock() *Genesis {
func DefaultYoloV2GenesisBlock() *Genesis {
// TODO: Update with yolov2 values + regenerate alloc data
return &Genesis{
Config: params.YoloV1ChainConfig,
Timestamp: 0x5ed754f1,
Config: params.YoloV2ChainConfig,
Timestamp: 0x5f91b932,
ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000008a37866fd3627c9205a37c8685666f32ec07bb1b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"),
GasLimit: 0x47b760,
Difficulty: big.NewInt(1),

57
core/rawdb/accessors_snapshot.go
View file

@ -17,6 +17,8 @@
package rawdb
import (
"encoding/binary"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
@ -118,3 +120,58 @@ func DeleteSnapshotJournal(db ethdb.KeyValueWriter) {
log.Crit("Failed to remove snapshot journal", "err", err)
}
}
// ReadSnapshotGenerator retrieves the serialized snapshot generator saved at
// the last shutdown.
func ReadSnapshotGenerator(db ethdb.KeyValueReader) []byte {
data, _ := db.Get(snapshotGeneratorKey)
return data
}
// WriteSnapshotGenerator stores the serialized snapshot generator to save at
// shutdown.
func WriteSnapshotGenerator(db ethdb.KeyValueWriter, generator []byte) {
if err := db.Put(snapshotGeneratorKey, generator); err != nil {
log.Crit("Failed to store snapshot generator", "err", err)
}
}
// DeleteSnapshotGenerator deletes the serialized snapshot generator saved at
// the last shutdown
func DeleteSnapshotGenerator(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotGeneratorKey); err != nil {
log.Crit("Failed to remove snapshot generator", "err", err)
}
}
// ReadSnapshotRecoveryNumber retrieves the block number of the last persisted
// snapshot layer.
func ReadSnapshotRecoveryNumber(db ethdb.KeyValueReader) *uint64 {
data, _ := db.Get(snapshotRecoveryKey)
if len(data) == 0 {
return nil
}
if len(data) != 8 {
return nil
}
number := binary.BigEndian.Uint64(data)
return &number
}
// WriteSnapshotRecoveryNumber stores the block number of the last persisted
// snapshot layer.
func WriteSnapshotRecoveryNumber(db ethdb.KeyValueWriter, number uint64) {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], number)
if err := db.Put(snapshotRecoveryKey, buf[:]); err != nil {
log.Crit("Failed to store snapshot recovery number", "err", err)
}
}
// DeleteSnapshotRecoveryNumber deletes the block number of the last persisted
// snapshot layer.
func DeleteSnapshotRecoveryNumber(db ethdb.KeyValueWriter) {
if err := db.Delete(snapshotRecoveryKey); err != nil {
log.Crit("Failed to remove snapshot recovery number", "err", err)
}
}

184
core/rawdb/chain_iterator.go
View file

@ -84,15 +84,17 @@ type blockTxHashes struct {
}
// iterateTransactions iterates over all transactions in the (canon) block
// number(s) given, and yields the hashes on a channel
func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool) (chan *blockTxHashes, chan struct{}) {
// number(s) given, and yields the hashes on a channel. If there is a signal
// received from interrupt channel, the iteration will be aborted and result
// channel will be closed.
func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes {
// One thread sequentially reads data from db
type numberRlp struct {
number uint64
rlp rlp.RawValue
}
if to == from {
return nil, nil
return nil
}
threads := to - from
if cpus := runtime.NumCPU(); threads > uint64(cpus) {
@ -101,7 +103,6 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
var (
rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel
hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
abortCh = make(chan struct{})
)
// lookup runs in one instance
lookup := func() {
@ -115,7 +116,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
// Feed the block to the aggregator, or abort on interrupt
select {
case rlpCh <- &numberRlp{n, data}:
case <-abortCh:
case <-interrupt:
return
}
if reverse {
@ -168,7 +169,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
// Feed the block to the aggregator, or abort on interrupt
select {
case hashesCh <- result:
case <-abortCh:
case <-interrupt:
return
}
}
@ -177,25 +178,28 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
for i := 0; i < int(threads); i++ {
go process()
}
return hashesCh, abortCh
return hashesCh
}
// IndexTransactions creates txlookup indices of the specified block range.
// indexTransactions creates txlookup indices of the specified block range.
//
// This function iterates canonical chain in reverse order, it has one main advantage:
// We can write tx index tail flag periodically even without the whole indexing
// procedure is finished. So that we can resume indexing procedure next time quickly.
func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
//
// There is a passed channel, the whole procedure will be interrupted if any
// signal received.
func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
// short circuit for invalid range
if from >= to {
return
}
var (
hashesCh, abortCh = iterateTransactions(db, from, to, true)
batch = db.NewBatch()
start = time.Now()
logged = start.Add(-7 * time.Second)
// Since we iterate in reverse, we expect the first number to come
hashesCh = iterateTransactions(db, from, to, true, interrupt)
batch = db.NewBatch()
start = time.Now()
logged = start.Add(-7 * time.Second)
// Since we iterate in reverse, we expect the first number to come
// in to be [to-1]. Therefore, setting lastNum to means that the
// prqueue gap-evaluation will work correctly
lastNum = to
@ -203,8 +207,6 @@ func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
// for stats reporting
blocks, txs = 0, 0
)
defer close(abortCh)
for chanDelivery := range hashesCh {
// Push the delivery into the queue and process contiguous ranges.
// Since we iterate in reverse, so lower numbers have lower prio, and
@ -215,6 +217,10 @@ func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
if _, priority := queue.Peek(); priority != int64(lastNum-1) {
break
}
// For testing
if hook != nil && !hook(lastNum-1) {
break
}
// Next block available, pop it off and index it
delivery := queue.PopItem().(*blockTxHashes)
lastNum = delivery.number
@ -223,8 +229,7 @@ func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
txs += len(delivery.hashes)
// If enough data was accumulated in memory or we're at the last block, dump to disk
if batch.ValueSize() > ethdb.IdealBatchSize {
// Also write the tail there
WriteTxIndexTail(batch, lastNum)
WriteTxIndexTail(batch, lastNum) // Also write the tail here
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
@ -238,67 +243,122 @@ func IndexTransactions(db ethdb.Database, from uint64, to uint64) {
}
}
}
if lastNum < to {
WriteTxIndexTail(batch, lastNum)
// No need to write the batch if we never entered the loop above...
// If there exists uncommitted data, flush them.
if batch.ValueSize() > 0 {
WriteTxIndexTail(batch, lastNum) // Also write the tail there
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
}
}
log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
select {
case <-interrupt:
log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
default:
log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
}
}
// UnindexTransactions removes txlookup indices of the specified block range.
func UnindexTransactions(db ethdb.Database, from uint64, to uint64) {
// IndexTransactions creates txlookup indices of the specified block range.
//
// This function iterates canonical chain in reverse order, it has one main advantage:
// We can write tx index tail flag periodically even without the whole indexing
// procedure is finished. So that we can resume indexing procedure next time quickly.
//
// There is a passed channel, the whole procedure will be interrupted if any
// signal received.
func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
indexTransactions(db, from, to, interrupt, nil)
}
// indexTransactionsForTesting is the internal debug version with an additional hook.
func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
indexTransactions(db, from, to, interrupt, hook)
}
// unindexTransactions removes txlookup indices of the specified block range.
//
// There is a passed channel, the whole procedure will be interrupted if any
// signal received.
func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
// short circuit for invalid range
if from >= to {
return
}
// Write flag first and then unindex the transaction indices. Some indices
// will be left in the database if crash happens but it's fine.
WriteTxIndexTail(db, to)
// If only one block is unindexed, do it directly
//if from+1 == to {
// data := ReadCanonicalBodyRLP(db, uint64(from))
// DeleteTxLookupEntries(db, ReadBlock(db, ReadCanonicalHash(db, from), from))
// log.Info("Unindexed transactions", "blocks", 1, "tail", to)
// return
//}
// TODO @holiman, add this back (if we want it)
var (
hashesCh, abortCh = iterateTransactions(db, from, to, false)
batch = db.NewBatch()
start = time.Now()
logged = start.Add(-7 * time.Second)
hashesCh = iterateTransactions(db, from, to, false, interrupt)
batch = db.NewBatch()
start = time.Now()
logged = start.Add(-7 * time.Second)
// we expect the first number to come in to be [from]. Therefore, setting
// nextNum to from means that the prqueue gap-evaluation will work correctly
nextNum = from
queue = prque.New(nil)
// for stats reporting
blocks, txs = 0, 0
)
defer close(abortCh)
// Otherwise spin up the concurrent iterator and unindexer
blocks, txs := 0, 0
for delivery := range hashesCh {
DeleteTxLookupEntries(batch, delivery.hashes)
txs += len(delivery.hashes)
blocks++
// If enough data was accumulated in memory or we're at the last block, dump to disk
// A batch counts the size of deletion as '1', so we need to flush more
// often than that.
if blocks%1000 == 0 {
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
// Push the delivery into the queue and process contiguous ranges.
queue.Push(delivery, -int64(delivery.number))
for !queue.Empty() {
// If the next available item is gapped, return
if _, priority := queue.Peek(); -priority != int64(nextNum) {
break
}
// For testing
if hook != nil && !hook(nextNum) {
break
}
delivery := queue.PopItem().(*blockTxHashes)
nextNum = delivery.number + 1
DeleteTxLookupEntries(batch, delivery.hashes)
txs += len(delivery.hashes)
blocks++
// If enough data was accumulated in memory or we're at the last block, dump to disk
// A batch counts the size of deletion as '1', so we need to flush more
// often than that.
if blocks%1000 == 0 {
WriteTxIndexTail(batch, nextNum)
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
}
batch.Reset()
}
// If we've spent too much time already, notify the user of what we're doing
if time.Since(logged) > 8*time.Second {
log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
batch.Reset()
}
// If we've spent too much time already, notify the user of what we're doing
if time.Since(logged) > 8*time.Second {
log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
}
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
// Commit the last batch if there exists uncommitted data
if batch.ValueSize() > 0 {
WriteTxIndexTail(batch, nextNum)
if err := batch.Write(); err != nil {
log.Crit("Failed writing batch to db", "error", err)
return
}
}
select {
case <-interrupt:
log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
default:
log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
}
log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
}
// UnindexTransactions removes txlookup indices of the specified block range.
//
// There is a passed channel, the whole procedure will be interrupted if any
// signal received.
func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
unindexTransactions(db, from, to, interrupt, nil)
}
// unindexTransactionsForTesting is the internal debug version with an additional hook.
func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
unindexTransactions(db, from, to, interrupt, hook)
}

85
core/rawdb/chain_iterator_test.go
View file

@ -20,6 +20,7 @@ import (
"math/big"
"reflect"
"sort"
"sync"
"testing"
"github.com/ethereum/go-ethereum/common"
@ -59,7 +60,7 @@ func TestChainIterator(t *testing.T) {
}
for i, c := range cases {
var numbers []int
hashCh, _ := iterateTransactions(chainDb, c.from, c.to, c.reverse)
hashCh := iterateTransactions(chainDb, c.from, c.to, c.reverse, nil)
if hashCh != nil {
for h := range hashCh {
numbers = append(numbers, int(h.number))
@ -80,3 +81,85 @@ func TestChainIterator(t *testing.T) {
}
}
}
func TestIndexTransactions(t *testing.T) {
// Construct test chain db
chainDb := NewMemoryDatabase()
var block *types.Block
var txs []*types.Transaction
for i := uint64(0); i <= 10; i++ {
if i == 0 {
block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil, newHasher()) // Empty genesis block
} else {
tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
txs = append(txs, tx)
block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher())
}
WriteBlock(chainDb, block)
WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
}
// verify checks whether the tx indices in the range [from, to)
// is expected.
verify := func(from, to int, exist bool, tail uint64) {
for i := from; i < to; i++ {
if i == 0 {
continue
}
number := ReadTxLookupEntry(chainDb, txs[i-1].Hash())
if exist && number == nil {
t.Fatalf("Transaction indice missing")
}
if !exist && number != nil {
t.Fatalf("Transaction indice is not deleted")
}
}
number := ReadTxIndexTail(chainDb)
if number == nil || *number != tail {
t.Fatalf("Transaction tail mismatch")
}
}
IndexTransactions(chainDb, 5, 11, nil)
verify(5, 11, true, 5)
verify(0, 5, false, 5)
IndexTransactions(chainDb, 0, 5, nil)
verify(0, 11, true, 0)
UnindexTransactions(chainDb, 0, 5, nil)
verify(5, 11, true, 5)
verify(0, 5, false, 5)
UnindexTransactions(chainDb, 5, 11, nil)
verify(0, 11, false, 11)
// Testing corner cases
signal := make(chan struct{})
var once sync.Once
indexTransactionsForTesting(chainDb, 5, 11, signal, func(n uint64) bool {
if n <= 8 {
once.Do(func() {
close(signal)
})
return false
}
return true
})
verify(9, 11, true, 9)
verify(0, 9, false, 9)
IndexTransactions(chainDb, 0, 9, nil)
signal = make(chan struct{})
var once2 sync.Once
unindexTransactionsForTesting(chainDb, 0, 11, signal, func(n uint64) bool {
if n >= 8 {
once2.Do(func() {
close(signal)
})
return false
}
return true
})
verify(8, 11, true, 8)
verify(0, 8, false, 8)
}

6
core/rawdb/schema.go
View file

@ -51,6 +51,12 @@ var (
// snapshotJournalKey tracks the in-memory diff layers across restarts.
snapshotJournalKey = []byte("SnapshotJournal")
// snapshotGeneratorKey tracks the snapshot generation marker across restarts.
snapshotGeneratorKey = []byte("SnapshotGenerator")
// snapshotRecoveryKey tracks the snapshot recovery marker across restarts.
snapshotRecoveryKey = []byte("SnapshotRecovery")
// txIndexTailKey tracks the oldest block whose transactions have been indexed.
txIndexTailKey = []byte("TransactionIndexTail")

136
core/state/access_list.go Normal file
View file

@ -0,0 +1,136 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package state
import (
"github.com/ethereum/go-ethereum/common"
)
type accessList struct {
addresses map[common.Address]int
slots []map[common.Hash]struct{}
}
// ContainsAddress returns true if the address is in the access list.
func (al *accessList) ContainsAddress(address common.Address) bool {
_, ok := al.addresses[address]
return ok
}
// Contains checks if a slot within an account is present in the access list, returning
// separate flags for the presence of the account and the slot respectively.
func (al *accessList) Contains(address common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) {
idx, ok := al.addresses[address]
if !ok {
// no such address (and hence zero slots)
return false, false
}
if idx == -1 {
// address yes, but no slots
return true, false
}
_, slotPresent = al.slots[idx][slot]
return true, slotPresent
}
// newAccessList creates a new accessList.
func newAccessList() *accessList {
return &accessList{
addresses: make(map[common.Address]int),
}
}
// Copy creates an independent copy of an accessList.
func (a *accessList) Copy() *accessList {
cp := newAccessList()
for k, v := range a.addresses {
cp.addresses[k] = v
}
cp.slots = make([]map[common.Hash]struct{}, len(a.slots))
for i, slotMap := range a.slots {
newSlotmap := make(map[common.Hash]struct{}, len(slotMap))
for k := range slotMap {
newSlotmap[k] = struct{}{}
}
cp.slots[i] = newSlotmap
}
return cp
}
// AddAddress adds an address to the access list, and returns 'true' if the operation
// caused a change (addr was not previously in the list).
func (al *accessList) AddAddress(address common.Address) bool {
if _, present := al.addresses[address]; present {
return false
}
al.addresses[address] = -1
return true
}
// AddSlot adds the specified (addr, slot) combo to the access list.
// Return values are:
// - address added
// - slot added
// For any 'true' value returned, a corresponding journal entry must be made.
func (al *accessList) AddSlot(address common.Address, slot common.Hash) (addrChange bool, slotChange bool) {
idx, addrPresent := al.addresses[address]
if !addrPresent || idx == -1 {
// Address not present, or addr present but no slots there
al.addresses[address] = len(al.slots)
slotmap := map[common.Hash]struct{}{slot: {}}
al.slots = append(al.slots, slotmap)
return !addrPresent, true
}
// There is already an (address,slot) mapping
slotmap := al.slots[idx]
if _, ok := slotmap[slot]; !ok {
slotmap[slot] = struct{}{}
// Journal add slot change
return false, true
}
// No changes required
return false, false
}
// DeleteSlot removes an (address, slot)-tuple from the access list.
// This operation needs to be performed in the same order as the addition happened.
// This method is meant to be used by the journal, which maintains ordering of
// operations.
func (al *accessList) DeleteSlot(address common.Address, slot common.Hash) {
idx, addrOk := al.addresses[address]
// There are two ways this can fail
if !addrOk {
panic("reverting slot change, address not present in list")
}
slotmap := al.slots[idx]
delete(slotmap, slot)
// If that was the last (first) slot, remove it
// Since additions and rollbacks are always performed in order,
// we can delete the item without worrying about screwing up later indices
if len(slotmap) == 0 {
al.slots = al.slots[:idx]
al.addresses[address] = -1
}
}
// DeleteAddress removes an address from the access list. This operation
// needs to be performed in the same order as the addition happened.
// This method is meant to be used by the journal, which maintains ordering of
// operations.
func (al *accessList) DeleteAddress(address common.Address) {
delete(al.addresses, address)
}

33
core/state/journal.go
View file

@ -130,6 +130,14 @@ type (
touchChange struct {
account *common.Address
}
// Changes to the access list
accessListAddAccountChange struct {
address *common.Address
}
accessListAddSlotChange struct {
address *common.Address
slot *common.Hash
}
)
func (ch createObjectChange) revert(s *StateDB) {
@ -234,3 +242,28 @@ func (ch addPreimageChange) revert(s *StateDB) {
func (ch addPreimageChange) dirtied() *common.Address {
return nil
}
func (ch accessListAddAccountChange) revert(s *StateDB) {
/*
One important invariant here, is that whenever a (addr, slot) is added, if the
addr is not already present, the add causes two journal entries:
- one for the address,
- one for the (address,slot)
Therefore, when unrolling the change, we can always blindly delete the
(addr) at this point, since no storage adds can remain when come upon
a single (addr) change.
*/
s.accessList.DeleteAddress(*ch.address)
}
func (ch accessListAddAccountChange) dirtied() *common.Address {
return nil
}
func (ch accessListAddSlotChange) revert(s *StateDB) {
s.accessList.DeleteSlot(*ch.address, *ch.slot)
}
func (ch accessListAddSlotChange) dirtied() *common.Address {
return nil
}

76
core/state/snapshot/disklayer_test.go
View file

@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/rlp"
)
// reverse reverses the contents of a byte slice. It's used to update random accs
@ -429,6 +430,81 @@ func TestDiskPartialMerge(t *testing.T) {
}
}
// Tests that when the bottom-most diff layer is merged into the disk
// layer whether the corresponding generator is persisted correctly.
func TestDiskGeneratorPersistence(t *testing.T) {
var (
accOne = randomHash()
accTwo = randomHash()
accOneSlotOne = randomHash()
accOneSlotTwo = randomHash()
accThree = randomHash()
accThreeSlot = randomHash()
baseRoot = randomHash()
diffRoot = randomHash()
diffTwoRoot = randomHash()
genMarker = append(randomHash().Bytes(), randomHash().Bytes()...)
)
// Testing scenario 1, the disk layer is still under the construction.
db := rawdb.NewMemoryDatabase()
rawdb.WriteAccountSnapshot(db, accOne, accOne[:])
rawdb.WriteStorageSnapshot(db, accOne, accOneSlotOne, accOneSlotOne[:])
rawdb.WriteStorageSnapshot(db, accOne, accOneSlotTwo, accOneSlotTwo[:])
rawdb.WriteSnapshotRoot(db, baseRoot)
// Create a disk layer based on all above updates
snaps := &Tree{
layers: map[common.Hash]snapshot{
baseRoot: &diskLayer{
diskdb: db,
cache: fastcache.New(500 * 1024),
root: baseRoot,
genMarker: genMarker,
},
},
}
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{
accTwo: accTwo[:],
}, nil); err != nil {
t.Fatalf("failed to update snapshot tree: %v", err)
}
if err := snaps.Cap(diffRoot, 0); err != nil {
t.Fatalf("failed to flatten snapshot tree: %v", err)
}
blob := rawdb.ReadSnapshotGenerator(db)
var generator journalGenerator
if err := rlp.DecodeBytes(blob, &generator); err != nil {
t.Fatalf("Failed to decode snapshot generator %v", err)
}
if !bytes.Equal(generator.Marker, genMarker) {
t.Fatalf("Generator marker is not matched")
}
// Test senario 2, the disk layer is fully generated
// Modify or delete some accounts, flatten everything onto disk
if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{
accThree: accThree.Bytes(),
}, map[common.Hash]map[common.Hash][]byte{
accThree: {accThreeSlot: accThreeSlot.Bytes()},
}); err != nil {
t.Fatalf("failed to update snapshot tree: %v", err)
}
diskLayer := snaps.layers[snaps.diskRoot()].(*diskLayer)
diskLayer.genMarker = nil // Construction finished
if err := snaps.Cap(diffTwoRoot, 0); err != nil {
t.Fatalf("failed to flatten snapshot tree: %v", err)
}
blob = rawdb.ReadSnapshotGenerator(db)
if err := rlp.DecodeBytes(blob, &generator); err != nil {
t.Fatalf("Failed to decode snapshot generator %v", err)
}
if len(generator.Marker) != 0 {
t.Fatalf("Failed to update snapshot generator")
}
}
// Tests that merging something into a disk layer persists it into the database
// and invalidates any previously written and cached values, discarding anything
// after the in-progress generation marker.

49
core/state/snapshot/generate.go
View file

@ -19,6 +19,7 @@ package snapshot
import (
"bytes"
"encoding/binary"
"fmt"
"math/big"
"time"
@ -112,9 +113,42 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i
genAbort: make(chan chan *generatorStats),
}
go base.generate(&generatorStats{wiping: wiper, start: time.Now()})
log.Debug("Start snapshot generation", "root", root)
return base
}
// journalProgress persists the generator stats into the database to resume later.
func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorStats) {
// Write out the generator marker. Note it's a standalone disk layer generator
// which is not mixed with journal. It's ok if the generator is persisted while
// journal is not.
entry := journalGenerator{
Done: marker == nil,
Marker: marker,
}
if stats != nil {
entry.Wiping = (stats.wiping != nil)
entry.Accounts = stats.accounts
entry.Slots = stats.slots
entry.Storage = uint64(stats.storage)
}
blob, err := rlp.EncodeToBytes(entry)
if err != nil {
panic(err) // Cannot happen, here to catch dev errors
}
var logstr string
switch len(marker) {
case 0:
logstr = "done"
case common.HashLength:
logstr = fmt.Sprintf("%#x", marker)
default:
logstr = fmt.Sprintf("%#x:%#x", marker[:common.HashLength], marker[common.HashLength:])
}
log.Debug("Journalled generator progress", "progress", logstr)
rawdb.WriteSnapshotGenerator(db, blob)
}
// generate is a background thread that iterates over the state and storage tries,
// constructing the state snapshot. All the arguments are purely for statistics
// gethering and logging, since the method surfs the blocks as they arrive, often
@ -186,11 +220,15 @@ func (dl *diskLayer) generate(stats *generatorStats) {
if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
// Only write and set the marker if we actually did something useful
if batch.ValueSize() > 0 {
// Ensure the generator entry is in sync with the data
marker := accountHash[:]
journalProgress(batch, marker, stats)
batch.Write()
batch.Reset()
dl.lock.Lock()
dl.genMarker = accountHash[:]
dl.genMarker = marker
dl.lock.Unlock()
}
if abort != nil {
@ -227,11 +265,15 @@ func (dl *diskLayer) generate(stats *generatorStats) {
if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
// Only write and set the marker if we actually did something useful
if batch.ValueSize() > 0 {
// Ensure the generator entry is in sync with the data
marker := append(accountHash[:], storeIt.Key...)
journalProgress(batch, marker, stats)
batch.Write()
batch.Reset()
dl.lock.Lock()
dl.genMarker = append(accountHash[:], storeIt.Key...)
dl.genMarker = marker
dl.lock.Unlock()
}
if abort != nil {
@ -263,6 +305,9 @@ func (dl *diskLayer) generate(stats *generatorStats) {
}
// Snapshot fully generated, set the marker to nil
if batch.ValueSize() > 0 {
// Ensure the generator entry is in sync with the data
journalProgress(batch, nil, stats)
batch.Write()
}
log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots,

244
core/state/snapshot/journal.go
View file

@ -33,6 +33,8 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
const journalVersion uint64 = 0
// journalGenerator is a disk layer entry containing the generator progress marker.
type journalGenerator struct {
Wiping bool // Whether the database was in progress of being wiped
@ -61,8 +63,91 @@ type journalStorage struct {
Vals [][]byte
}
// loadAndParseLegacyJournal tries to parse the snapshot journal in legacy format.
func loadAndParseLegacyJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) {
// Retrieve the journal, for legacy journal it must exist since even for
// 0 layer it stores whether we've already generated the snapshot or are
// in progress only.
journal := rawdb.ReadSnapshotJournal(db)
if len(journal) == 0 {
return nil, journalGenerator{}, errors.New("missing or corrupted snapshot journal")
}
r := rlp.NewStream(bytes.NewReader(journal), 0)
// Read the snapshot generation progress for the disk layer
var generator journalGenerator
if err := r.Decode(&generator); err != nil {
return nil, journalGenerator{}, fmt.Errorf("failed to load snapshot progress marker: %v", err)
}
// Load all the snapshot diffs from the journal
snapshot, err := loadDiffLayer(base, r)
if err != nil {
return nil, generator, err
}
return snapshot, generator, nil
}
// loadAndParseJournal tries to parse the snapshot journal in latest format.
func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) {
// Retrieve the disk layer generator. It must exist, no matter the
// snapshot is fully generated or not. Otherwise the entire disk
// layer is invalid.
generatorBlob := rawdb.ReadSnapshotGenerator(db)
if len(generatorBlob) == 0 {
return nil, journalGenerator{}, errors.New("missing snapshot generator")
}
var generator journalGenerator
if err := rlp.DecodeBytes(generatorBlob, &generator); err != nil {
return nil, journalGenerator{}, fmt.Errorf("failed to decode snapshot generator: %v", err)
}
// Retrieve the diff layer journal. It's possible that the journal is
// not existent, e.g. the disk layer is generating while that the Geth
// crashes without persisting the diff journal.
// So if there is no journal, or the journal is invalid(e.g. the journal
// is not matched with disk layer; or the it's the legacy-format journal,
// etc.), we just discard all diffs and try to recover them later.
journal := rawdb.ReadSnapshotJournal(db)
if len(journal) == 0 {
log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing")
return base, generator, nil
}
r := rlp.NewStream(bytes.NewReader(journal), 0)
// Firstly, resolve the first element as the journal version
version, err := r.Uint()
if err != nil {
log.Warn("Failed to resolve the journal version", "error", err)
return base, generator, nil
}
if version != journalVersion {
log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version)
return base, generator, nil
}
// Secondly, resolve the disk layer root, ensure it's continuous
// with disk layer. Note now we can ensure it's the snapshot journal
// correct version, so we expect everything can be resolved properly.
var root common.Hash
if err := r.Decode(&root); err != nil {
return nil, journalGenerator{}, errors.New("missing disk layer root")
}
// The diff journal is not matched with disk, discard them.
// It can happen that Geth crashes without persisting the latest
// diff journal.
if !bytes.Equal(root.Bytes(), base.root.Bytes()) {
log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched")
return base, generator, nil
}
// Load all the snapshot diffs from the journal
snapshot, err := loadDiffLayer(base, r)
if err != nil {
return nil, journalGenerator{}, err
}
log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root())
return snapshot, generator, nil
}
// loadSnapshot loads a pre-existing state snapshot backed by a key-value store.
func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) (snapshot, error) {
func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, error) {
// Retrieve the block number and hash of the snapshot, failing if no snapshot
// is present in the database (or crashed mid-update).
baseRoot := rawdb.ReadSnapshotRoot(diskdb)
@ -75,28 +160,36 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
cache: fastcache.New(cache * 1024 * 1024),
root: baseRoot,
}
// Retrieve the journal, it must exist since even for 0 layer it stores whether
// we've already generated the snapshot or are in progress only
journal := rawdb.ReadSnapshotJournal(diskdb)
if len(journal) == 0 {
return nil, errors.New("missing or corrupted snapshot journal")
var legacy bool
snapshot, generator, err := loadAndParseJournal(diskdb, base)
if err != nil {
log.Warn("Failed to load new-format journal", "error", err)
snapshot, generator, err = loadAndParseLegacyJournal(diskdb, base)
legacy = true
}
r := rlp.NewStream(bytes.NewReader(journal), 0)
// Read the snapshot generation progress for the disk layer
var generator journalGenerator
if err := r.Decode(&generator); err != nil {
return nil, fmt.Errorf("failed to load snapshot progress marker: %v", err)
}
// Load all the snapshot diffs from the journal
snapshot, err := loadDiffLayer(base, r)
if err != nil {
return nil, err
}
// Entire snapshot journal loaded, sanity check the head and return
// Journal doesn't exist, don't worry if it's not supposed to
// Entire snapshot journal loaded, sanity check the head. If the loaded
// snapshot is not matched with current state root, print a warning log
// or discard the entire snapshot it's legacy snapshot.
//
// Possible scenario: Geth was crashed without persisting journal and then
// restart, the head is rewound to the point with available state(trie)
// which is below the snapshot. In this case the snapshot can be recovered
// by re-executing blocks but right now it's unavailable.
if head := snapshot.Root(); head != root {
return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root)
// If it's legacy snapshot, or it's new-format snapshot but
// it's not in recovery mode, returns the error here for
// rebuilding the entire snapshot forcibly.
if legacy || !recovery {
return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root)
}
// It's in snapshot recovery, the assumption is held that
// the disk layer is always higher than chain head. It can
// be eventually recovered when the chain head beyonds the
// disk layer.
log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root)
}
// Everything loaded correctly, resume any suspended operations
if !generator.Done {
@ -183,8 +276,8 @@ func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) {
return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r)
}
// Journal writes the persistent layer generator stats into a buffer to be stored
// in the database as the snapshot journal.
// Journal terminates any in-progress snapshot generation, also implicitly pushing
// the progress into the database.
func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
// If the snapshot is currently being generated, abort it
var stats *generatorStats
@ -203,20 +296,10 @@ func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
if dl.stale {
return common.Hash{}, ErrSnapshotStale
}
// Write out the generator marker
entry := journalGenerator{
Done: dl.genMarker == nil,
Marker: dl.genMarker,
}
if stats != nil {
entry.Wiping = (stats.wiping != nil)
entry.Accounts = stats.accounts
entry.Slots = stats.slots
entry.Storage = uint64(stats.storage)
}
if err := rlp.Encode(buffer, entry); err != nil {
return common.Hash{}, err
}
// Ensure the generator stats is written even if none was ran this cycle
journalProgress(dl.diskdb, dl.genMarker, stats)
log.Debug("Journalled disk layer", "root", dl.root)
return dl.root, nil
}
@ -266,5 +349,98 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
if err := rlp.Encode(buffer, storage); err != nil {
return common.Hash{}, err
}
log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root())
return base, nil
}
// LegacyJournal writes the persistent layer generator stats into a buffer
// to be stored in the database as the snapshot journal.
//
// Note it's the legacy version which is only used in testing right now.
func (dl *diskLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) {
// If the snapshot is currently being generated, abort it
var stats *generatorStats
if dl.genAbort != nil {
abort := make(chan *generatorStats)
dl.genAbort <- abort
if stats = <-abort; stats != nil {
stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker)
}
}
// Ensure the layer didn't get stale
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.stale {
return common.Hash{}, ErrSnapshotStale
}
// Write out the generator marker
entry := journalGenerator{
Done: dl.genMarker == nil,
Marker: dl.genMarker,
}
if stats != nil {
entry.Wiping = (stats.wiping != nil)
entry.Accounts = stats.accounts
entry.Slots = stats.slots
entry.Storage = uint64(stats.storage)
}
log.Debug("Legacy journalled disk layer", "root", dl.root)
if err := rlp.Encode(buffer, entry); err != nil {
return common.Hash{}, err
}
return dl.root, nil
}
// Journal writes the memory layer contents into a buffer to be stored in the
// database as the snapshot journal.
//
// Note it's the legacy version which is only used in testing right now.
func (dl *diffLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) {
// Journal the parent first
base, err := dl.parent.LegacyJournal(buffer)
if err != nil {
return common.Hash{}, err
}
// Ensure the layer didn't get stale
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.Stale() {
return common.Hash{}, ErrSnapshotStale
}
// Everything below was journalled, persist this layer too
if err := rlp.Encode(buffer, dl.root); err != nil {
return common.Hash{}, err
}
destructs := make([]journalDestruct, 0, len(dl.destructSet))
for hash := range dl.destructSet {
destructs = append(destructs, journalDestruct{Hash: hash})
}
if err := rlp.Encode(buffer, destructs); err != nil {
return common.Hash{}, err
}
accounts := make([]journalAccount, 0, len(dl.accountData))
for hash, blob := range dl.accountData {
accounts = append(accounts, journalAccount{Hash: hash, Blob: blob})
}
if err := rlp.Encode(buffer, accounts); err != nil {
return common.Hash{}, err
}
storage := make([]journalStorage, 0, len(dl.storageData))
for hash, slots := range dl.storageData {
keys := make([]common.Hash, 0, len(slots))
vals := make([][]byte, 0, len(slots))
for key, val := range slots {
keys = append(keys, key)
vals = append(vals, val)
}
storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals})
}
if err := rlp.Encode(buffer, storage); err != nil {
return common.Hash{}, err
}
log.Debug("Legacy journalled disk layer", "root", dl.root, "parent", dl.parent.Root())
return base, nil
}

157
core/state/snapshot/snapshot.go
View file

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
@ -86,6 +87,10 @@ var (
// range of accounts covered.
ErrNotCoveredYet = errors.New("not covered yet")
// ErrNotConstructed is returned if the callers want to iterate the snapshot
// while the generation is not finished yet.
ErrNotConstructed = errors.New("snapshot is not constructed")
// errSnapshotCycle is returned if a snapshot is attempted to be inserted
// that forms a cycle in the snapshot tree.
errSnapshotCycle = errors.New("snapshot cycle")
@ -132,6 +137,10 @@ type snapshot interface {
// flattening everything down (bad for reorgs).
Journal(buffer *bytes.Buffer) (common.Hash, error)
// LegacyJournal is basically identical to Journal. it's the legacy version for
// flushing legacy journal. Now the only purpose of this function is for testing.
LegacyJournal(buffer *bytes.Buffer) (common.Hash, error)
// Stale return whether this layer has become stale (was flattened across) or
// if it's still live.
Stale() bool
@ -164,10 +173,12 @@ type Tree struct {
// store (with a number of memory layers from a journal), ensuring that the head
// of the snapshot matches the expected one.
//
// If the snapshot is missing or inconsistent, the entirety is deleted and will
// be reconstructed from scratch based on the tries in the key-value store, on a
// background thread.
func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool) *Tree {
// If the snapshot is missing or the disk layer is broken, the entire is deleted
// and will be reconstructed from scratch based on the tries in the key-value
// store, on a background thread. If the memory layers from the journal is not
// continuous with disk layer or the journal is missing, all diffs will be discarded
// iff it's in "recovery" mode, otherwise rebuild is mandatory.
func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool, recovery bool) *Tree {
// Create a new, empty snapshot tree
snap := &Tree{
diskdb: diskdb,
@ -179,7 +190,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm
defer snap.waitBuild()
}
// Attempt to load a previously persisted snapshot and rebuild one if failed
head, err := loadSnapshot(diskdb, triedb, cache, root)
head, err := loadSnapshot(diskdb, triedb, cache, root, recovery)
if err != nil {
log.Warn("Failed to load snapshot, regenerating", "err", err)
snap.Rebuild(root)
@ -194,7 +205,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm
}
// waitBuild blocks until the snapshot finishes rebuilding. This method is meant
// to be used by tests to ensure we're testing what we believe we are.
// to be used by tests to ensure we're testing what we believe we are.
func (t *Tree) waitBuild() {
// Find the rebuild termination channel
var done chan struct{}
@ -411,6 +422,9 @@ func (t *Tree) cap(diff *diffLayer, layers int) *diskLayer {
// diffToDisk merges a bottom-most diff into the persistent disk layer underneath
// it. The method will panic if called onto a non-bottom-most diff layer.
//
// The disk layer persistence should be operated in an atomic way. All updates should
// be discarded if the whole transition if not finished.
func diffToDisk(bottom *diffLayer) *diskLayer {
var (
base = bottom.parent.(*diskLayer)
@ -423,8 +437,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
base.genAbort <- abort
stats = <-abort
}
// Start by temporarily deleting the current snapshot block marker. This
// ensures that in the case of a crash, the entire snapshot is invalidated.
// Put the deletion in the batch writer, flush all updates in the final step.
rawdb.DeleteSnapshotRoot(batch)
// Mark the original base as stale as we're going to create a new wrapper
@ -467,12 +480,6 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
base.cache.Set(hash[:], data)
snapshotCleanAccountWriteMeter.Mark(int64(len(data)))
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write account snapshot", "err", err)
}
batch.Reset()
}
snapshotFlushAccountItemMeter.Mark(1)
snapshotFlushAccountSizeMeter.Mark(int64(len(data)))
}
@ -501,18 +508,19 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
snapshotFlushStorageItemMeter.Mark(1)
snapshotFlushStorageSizeMeter.Mark(int64(len(data)))
}
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
log.Crit("Failed to write storage snapshot", "err", err)
}
batch.Reset()
}
}
// Update the snapshot block marker and write any remainder data
rawdb.WriteSnapshotRoot(batch, bottom.root)
// Write out the generator progress marker and report
journalProgress(batch, base.genMarker, stats)
// Flush all the updates in the single db operation. Ensure the
// disk layer transition is atomic.
if err := batch.Write(); err != nil {
log.Crit("Failed to write leftover snapshot", "err", err)
}
log.Debug("Journalled disk layer", "root", bottom.root, "complete", base.genMarker == nil)
res := &diskLayer{
root: bottom.root,
cache: base.cache,
@ -550,7 +558,21 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) {
t.lock.Lock()
defer t.lock.Unlock()
// Firstly write out the metadata of journal
journal := new(bytes.Buffer)
if err := rlp.Encode(journal, journalVersion); err != nil {
return common.Hash{}, err
}
diskroot := t.diskRoot()
if diskroot == (common.Hash{}) {
return common.Hash{}, errors.New("invalid disk root")
}
// Secondly write out the disk layer root, ensure the
// diff journal is continuous with disk.
if err := rlp.Encode(journal, diskroot); err != nil {
return common.Hash{}, err
}
// Finally write out the journal of each layer in reverse order.
base, err := snap.(snapshot).Journal(journal)
if err != nil {
return common.Hash{}, err
@ -560,6 +582,29 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) {
return base, nil
}
// LegacyJournal is basically identical to Journal. it's the legacy
// version for flushing legacy journal. Now the only purpose of this
// function is for testing.
func (t *Tree) LegacyJournal(root common.Hash) (common.Hash, error) {
// Retrieve the head snapshot to journal from var snap snapshot
snap := t.Snapshot(root)
if snap == nil {
return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root)
}
// Run the journaling
t.lock.Lock()
defer t.lock.Unlock()
journal := new(bytes.Buffer)
base, err := snap.(snapshot).LegacyJournal(journal)
if err != nil {
return common.Hash{}, err
}
// Store the journal into the database and return
rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes())
return base, nil
}
// Rebuild wipes all available snapshot data from the persistent database and
// discard all caches and diff layers. Afterwards, it starts a new snapshot
// generator with the given root hash.
@ -567,6 +612,10 @@ func (t *Tree) Rebuild(root common.Hash) {
t.lock.Lock()
defer t.lock.Unlock()
// Firstly delete any recovery flag in the database. Because now we are
// building a brand new snapshot.
rawdb.DeleteSnapshotRecoveryNumber(t.diskdb)
// Track whether there's a wipe currently running and keep it alive if so
var wiper chan struct{}
@ -609,11 +658,79 @@ func (t *Tree) Rebuild(root common.Hash) {
// AccountIterator creates a new account iterator for the specified root hash and
// seeks to a starting account hash.
func (t *Tree) AccountIterator(root common.Hash, seek common.Hash) (AccountIterator, error) {
ok, err := t.generating()
if err != nil {
return nil, err
}
if ok {
return nil, ErrNotConstructed
}
return newFastAccountIterator(t, root, seek)
}
// StorageIterator creates a new storage iterator for the specified root hash and
// account. The iterator will be move to the specific start position.
func (t *Tree) StorageIterator(root common.Hash, account common.Hash, seek common.Hash) (StorageIterator, error) {
ok, err := t.generating()
if err != nil {
return nil, err
}
if ok {
return nil, ErrNotConstructed
}
return newFastStorageIterator(t, root, account, seek)
}
// disklayer is an internal helper function to return the disk layer.
// The lock of snapTree is assumed to be held already.
func (t *Tree) disklayer() *diskLayer {
var snap snapshot
for _, s := range t.layers {
snap = s
break
}
if snap == nil {
return nil
}
switch layer := snap.(type) {
case *diskLayer:
return layer
case *diffLayer:
return layer.origin
default:
panic(fmt.Sprintf("%T: undefined layer", snap))
}
}
// diskRoot is a internal helper function to return the disk layer root.
// The lock of snapTree is assumed to be held already.
func (t *Tree) diskRoot() common.Hash {
disklayer := t.disklayer()
if disklayer == nil {
return common.Hash{}
}
return disklayer.Root()
}
// generating is an internal helper function which reports whether the snapshot
// is still under the construction.
func (t *Tree) generating() (bool, error) {
t.lock.Lock()
defer t.lock.Unlock()
layer := t.disklayer()
if layer == nil {
return false, errors.New("disk layer is missing")
}
layer.lock.RLock()
defer layer.lock.RUnlock()
return layer.genMarker != nil, nil
}
// diskRoot is a external helper function to return the disk layer root.
func (t *Tree) DiskRoot() common.Hash {
t.lock.Lock()
defer t.lock.Unlock()
return t.diskRoot()
}

55
core/state/statedb.go
View file

@ -93,6 +93,9 @@ type StateDB struct {
preimages map[common.Hash][]byte
// Per-transaction access list
accessList *accessList
// Journal of state modifications. This is the backbone of
// Snapshot and RevertToSnapshot.
journal *journal
@ -129,6 +132,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error)
logs: make(map[common.Hash][]*types.Log),
preimages: make(map[common.Hash][]byte),
journal: newJournal(),
accessList: newAccessList(),
}
if sdb.snaps != nil {
if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil {
@ -178,6 +182,7 @@ func (s *StateDB) Reset(root common.Hash) error {
s.snapStorage = make(map[common.Hash]map[common.Hash][]byte)
}
}
s.accessList = newAccessList()
return nil
}
@ -697,6 +702,12 @@ func (s *StateDB) Copy() *StateDB {
for hash, preimage := range s.preimages {
state.preimages[hash] = preimage
}
// Do we need to copy the access list? In practice: No. At the start of a
// transaction, the access list is empty. In practice, we only ever copy state
// _between_ transactions/blocks, never in the middle of a transaction.
// However, it doesn't cost us much to copy an empty list, so we do it anyway
// to not blow up if we ever decide copy it in the middle of a transaction
state.accessList = s.accessList.Copy()
return state
}
@ -798,6 +809,7 @@ func (s *StateDB) Prepare(thash, bhash common.Hash, ti int) {
s.thash = thash
s.bhash = bhash
s.txIndex = ti
s.accessList = newAccessList()
}
func (s *StateDB) clearJournalAndRefund() {
@ -869,11 +881,50 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil {
log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err)
}
if err := s.snaps.Cap(root, 127); err != nil { // Persistent layer is 128th, the last available trie
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 127, "err", err)
// Keep 128 diff layers in the memory, persistent layer is 129th.
// - head layer is paired with HEAD state
// - head-1 layer is paired with HEAD-1 state
// - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state
if err := s.snaps.Cap(root, 128); err != nil {
log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err)
}
}
s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil
}
return root, err
}
// AddAddressToAccessList adds the given address to the access list
func (s *StateDB) AddAddressToAccessList(addr common.Address) {
if s.accessList.AddAddress(addr) {
s.journal.append(accessListAddAccountChange{&addr})
}
}
// AddSlotToAccessList adds the given (address, slot)-tuple to the access list
func (s *StateDB) AddSlotToAccessList(addr common.Address, slot common.Hash) {
addrMod, slotMod := s.accessList.AddSlot(addr, slot)
if addrMod {
// In practice, this should not happen, since there is no way to enter the
// scope of 'address' without having the 'address' become already added
// to the access list (via call-variant, create, etc).
// Better safe than sorry, though
s.journal.append(accessListAddAccountChange{&addr})
}
if slotMod {
s.journal.append(accessListAddSlotChange{
address: &addr,
slot: &slot,
})
}
}
// AddressInAccessList returns true if the given address is in the access list.
func (s *StateDB) AddressInAccessList(addr common.Address) bool {
return s.accessList.ContainsAddress(addr)
}
// SlotInAccessList returns true if the given (address, slot)-tuple is in the access list.
func (s *StateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addressPresent bool, slotPresent bool) {
return s.accessList.Contains(addr, slot)
}

188
core/state/statedb_test.go
View file

@ -328,6 +328,20 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction {
},
args: make([]int64, 1),
},
{
name: "AddAddressToAccessList",
fn: func(a testAction, s *StateDB) {
s.AddAddressToAccessList(addr)
},
},
{
name: "AddSlotToAccessList",
fn: func(a testAction, s *StateDB) {
s.AddSlotToAccessList(addr,
common.Hash{byte(a.args[0])})
},
args: make([]int64, 1),
},
}
action := actions[r.Intn(len(actions))]
var nameargs []string
@ -727,3 +741,177 @@ func TestMissingTrieNodes(t *testing.T) {
t.Fatalf("expected error, got root :%x", root)
}
}
func TestStateDBAccessList(t *testing.T) {
// Some helpers
addr := func(a string) common.Address {
return common.HexToAddress(a)
}
slot := func(a string) common.Hash {
return common.HexToHash(a)
}
memDb := rawdb.NewMemoryDatabase()
db := NewDatabase(memDb)
state, _ := New(common.Hash{}, db, nil)
state.accessList = newAccessList()
verifyAddrs := func(astrings ...string) {
t.Helper()
// convert to common.Address form
var addresses []common.Address
var addressMap = make(map[common.Address]struct{})
for _, astring := range astrings {
address := addr(astring)
addresses = append(addresses, address)
addressMap[address] = struct{}{}
}
// Check that the given addresses are in the access list
for _, address := range addresses {
if !state.AddressInAccessList(address) {
t.Fatalf("expected %x to be in access list", address)
}
}
// Check that only the expected addresses are present in the acesslist
for address := range state.accessList.addresses {
if _, exist := addressMap[address]; !exist {
t.Fatalf("extra address %x in access list", address)
}
}
}
verifySlots := func(addrString string, slotStrings ...string) {
if !state.AddressInAccessList(addr(addrString)) {
t.Fatalf("scope missing address/slots %v", addrString)
}
var address = addr(addrString)
// convert to common.Hash form
var slots []common.Hash
var slotMap = make(map[common.Hash]struct{})
for _, slotString := range slotStrings {
s := slot(slotString)
slots = append(slots, s)
slotMap[s] = struct{}{}
}
// Check that the expected items are in the access list
for i, s := range slots {
if _, slotPresent := state.SlotInAccessList(address, s); !slotPresent {
t.Fatalf("input %d: scope missing slot %v (address %v)", i, s, addrString)
}
}
// Check that no extra elements are in the access list
index := state.accessList.addresses[address]
if index >= 0 {
stateSlots := state.accessList.slots[index]
for s := range stateSlots {
if _, slotPresent := slotMap[s]; !slotPresent {
t.Fatalf("scope has extra slot %v (address %v)", s, addrString)
}
}
}
}
state.AddAddressToAccessList(addr("aa")) // 1
state.AddSlotToAccessList(addr("bb"), slot("01")) // 2,3
state.AddSlotToAccessList(addr("bb"), slot("02")) // 4
verifyAddrs("aa", "bb")
verifySlots("bb", "01", "02")
// Make a copy
stateCopy1 := state.Copy()
if exp, got := 4, state.journal.length(); exp != got {
t.Fatalf("journal length mismatch: have %d, want %d", got, exp)
}
// same again, should cause no journal entries
state.AddSlotToAccessList(addr("bb"), slot("01"))
state.AddSlotToAccessList(addr("bb"), slot("02"))
state.AddAddressToAccessList(addr("aa"))
if exp, got := 4, state.journal.length(); exp != got {
t.Fatalf("journal length mismatch: have %d, want %d", got, exp)
}
// some new ones
state.AddSlotToAccessList(addr("bb"), slot("03")) // 5
state.AddSlotToAccessList(addr("aa"), slot("01")) // 6
state.AddSlotToAccessList(addr("cc"), slot("01")) // 7,8
state.AddAddressToAccessList(addr("cc"))
if exp, got := 8, state.journal.length(); exp != got {
t.Fatalf("journal length mismatch: have %d, want %d", got, exp)
}
verifyAddrs("aa", "bb", "cc")
verifySlots("aa", "01")
verifySlots("bb", "01", "02", "03")
verifySlots("cc", "01")
// now start rolling back changes
state.journal.revert(state, 7)
if _, ok := state.SlotInAccessList(addr("cc"), slot("01")); ok {
t.Fatalf("slot present, expected missing")
}
verifyAddrs("aa", "bb", "cc")
verifySlots("aa", "01")
verifySlots("bb", "01", "02", "03")
state.journal.revert(state, 6)
if state.AddressInAccessList(addr("cc")) {
t.Fatalf("addr present, expected missing")
}
verifyAddrs("aa", "bb")
verifySlots("aa", "01")
verifySlots("bb", "01", "02", "03")
state.journal.revert(state, 5)
if _, ok := state.SlotInAccessList(addr("aa"), slot("01")); ok {
t.Fatalf("slot present, expected missing")
}
verifyAddrs("aa", "bb")
verifySlots("bb", "01", "02", "03")
state.journal.revert(state, 4)
if _, ok := state.SlotInAccessList(addr("bb"), slot("03")); ok {
t.Fatalf("slot present, expected missing")
}
verifyAddrs("aa", "bb")
verifySlots("bb", "01", "02")
state.journal.revert(state, 3)
if _, ok := state.SlotInAccessList(addr("bb"), slot("02")); ok {
t.Fatalf("slot present, expected missing")
}
verifyAddrs("aa", "bb")
verifySlots("bb", "01")
state.journal.revert(state, 2)
if _, ok := state.SlotInAccessList(addr("bb"), slot("01")); ok {
t.Fatalf("slot present, expected missing")
}
verifyAddrs("aa", "bb")
state.journal.revert(state, 1)
if state.AddressInAccessList(addr("bb")) {
t.Fatalf("addr present, expected missing")
}
verifyAddrs("aa")
state.journal.revert(state, 0)
if state.AddressInAccessList(addr("aa")) {
t.Fatalf("addr present, expected missing")
}
if got, exp := len(state.accessList.addresses), 0; got != exp {
t.Fatalf("expected empty, got %d", got)
}
if got, exp := len(state.accessList.slots), 0; got != exp {
t.Fatalf("expected empty, got %d", got)
}
// Check the copy
// Make a copy
state = stateCopy1
verifyAddrs("aa", "bb")
verifySlots("bb", "01", "02")
if got, exp := len(state.accessList.addresses), 2; got != exp {
t.Fatalf("expected empty, got %d", got)
}
if got, exp := len(state.accessList.slots), 1; got != exp {
t.Fatalf("expected empty, got %d", got)
}
}

12
core/state_processor.go
View file

@ -95,6 +95,18 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo
// Create a new environment which holds all relevant information
// about the transaction and calling mechanisms.
vmenv := vm.NewEVM(context, statedb, config, cfg)
if config.IsYoloV2(header.Number) {
statedb.AddAddressToAccessList(msg.From())
if dst := msg.To(); dst != nil {
statedb.AddAddressToAccessList(*dst)
// If it's a create-tx, the destination will be added inside evm.create
}
for _, addr := range vmenv.ActivePrecompiles() {
statedb.AddAddressToAccessList(addr)
}
}
// Apply the transaction to the current state (included in the env)
result, err := ApplyMessage(vmenv, msg, gp)
if err != nil {

20
core/types/derive_sha.go
View file

@ -17,8 +17,6 @@
package types
import (
"bytes"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
)
@ -37,26 +35,24 @@ type Hasher interface {
func DeriveSha(list DerivableList, hasher Hasher) common.Hash {
hasher.Reset()
keybuf := new(bytes.Buffer)
// StackTrie requires values to be inserted in increasing
// hash order, which is not the order that `list` provides
// hashes in. This insertion sequence ensures that the
// order is correct.
var buf []byte
for i := 1; i < list.Len() && i <= 0x7f; i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
buf = rlp.AppendUint64(buf[:0], uint64(i))
hasher.Update(buf, list.GetRlp(i))
}
if list.Len() > 0 {
keybuf.Reset()
rlp.Encode(keybuf, uint(0))
hasher.Update(keybuf.Bytes(), list.GetRlp(0))
buf = rlp.AppendUint64(buf[:0], 0)
hasher.Update(buf, list.GetRlp(0))
}
for i := 0x80; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
buf = rlp.AppendUint64(buf[:0], uint64(i))
hasher.Update(buf, list.GetRlp(i))
}
return hasher.Hash()
}

28
core/vm/contracts.go
View file

@ -78,9 +78,9 @@ var PrecompiledContractsIstanbul = map[common.Address]PrecompiledContract{
common.BytesToAddress([]byte{9}): &blake2F{},
}
// PrecompiledContractsYoloV1 contains the default set of pre-compiled Ethereum
// contracts used in the Yolo v1 test release.
var PrecompiledContractsYoloV1 = map[common.Address]PrecompiledContract{
// PrecompiledContractsYoloV2 contains the default set of pre-compiled Ethereum
// contracts used in the Yolo v2 test release.
var PrecompiledContractsYoloV2 = map[common.Address]PrecompiledContract{
common.BytesToAddress([]byte{1}): &ecrecover{},
common.BytesToAddress([]byte{2}): &sha256hash{},
common.BytesToAddress([]byte{3}): &ripemd160hash{},
@ -101,6 +101,28 @@ var PrecompiledContractsYoloV1 = map[common.Address]PrecompiledContract{
common.BytesToAddress([]byte{18}): &bls12381MapG2{},
}
var (
PrecompiledAddressesYoloV2 []common.Address
PrecompiledAddressesIstanbul []common.Address
PrecompiledAddressesByzantium []common.Address
PrecompiledAddressesHomestead []common.Address
)
func init() {
for k := range PrecompiledContractsHomestead {
PrecompiledAddressesHomestead = append(PrecompiledAddressesHomestead, k)
}
for k := range PrecompiledContractsByzantium {
PrecompiledAddressesHomestead = append(PrecompiledAddressesByzantium, k)
}
for k := range PrecompiledContractsIstanbul {
PrecompiledAddressesIstanbul = append(PrecompiledAddressesIstanbul, k)
}
for k := range PrecompiledContractsYoloV2 {
PrecompiledAddressesYoloV2 = append(PrecompiledAddressesYoloV2, k)
}
}
// RunPrecompiledContract runs and evaluates the output of a precompiled contract.
// It returns
// - the returned bytes,

2
core/vm/contracts_test.go
View file

@ -43,7 +43,7 @@ type precompiledFailureTest struct {
Name string
}
var allPrecompiles = PrecompiledContractsYoloV1
var allPrecompiles = PrecompiledContractsYoloV2
// EIP-152 test vectors
var blake2FMalformedInputTests = []precompiledFailureTest{

39
core/vm/eips.go
View file

@ -25,6 +25,7 @@ import (
)
var activators = map[int]func(*JumpTable){
2929: enable2929,
2200: enable2200,
1884: enable1884,
1344: enable1344,
@ -134,3 +135,41 @@ func enable2315(jt *JumpTable) {
jumps: true,
}
}
// enable2929 enables "EIP-2929: Gas cost increases for state access opcodes"
// https://eips.ethereum.org/EIPS/eip-2929
func enable2929(jt *JumpTable) {
jt[SSTORE].dynamicGas = gasSStoreEIP2929
jt[SLOAD].constantGas = 0
jt[SLOAD].dynamicGas = gasSLoadEIP2929
jt[EXTCODECOPY].constantGas = WarmStorageReadCostEIP2929
jt[EXTCODECOPY].dynamicGas = gasExtCodeCopyEIP2929
jt[EXTCODESIZE].constantGas = WarmStorageReadCostEIP2929
jt[EXTCODESIZE].dynamicGas = gasEip2929AccountCheck
jt[EXTCODEHASH].constantGas = WarmStorageReadCostEIP2929
jt[EXTCODEHASH].dynamicGas = gasEip2929AccountCheck
jt[BALANCE].constantGas = WarmStorageReadCostEIP2929
jt[BALANCE].dynamicGas = gasEip2929AccountCheck
jt[CALL].constantGas = WarmStorageReadCostEIP2929
jt[CALL].dynamicGas = gasCallEIP2929
jt[CALLCODE].constantGas = WarmStorageReadCostEIP2929
jt[CALLCODE].dynamicGas = gasCallCodeEIP2929
jt[STATICCALL].constantGas = WarmStorageReadCostEIP2929
jt[STATICCALL].dynamicGas = gasStaticCallEIP2929
jt[DELEGATECALL].constantGas = WarmStorageReadCostEIP2929
jt[DELEGATECALL].dynamicGas = gasDelegateCallEIP2929
// This was previously part of the dynamic cost, but we're using it as a constantGas
// factor here
jt[SELFDESTRUCT].constantGas = params.SelfdestructGasEIP150
jt[SELFDESTRUCT].dynamicGas = gasSelfdestructEIP2929
}

25
core/vm/evm.go
View file

@ -42,11 +42,26 @@ type (
GetHashFunc func(uint64) common.Hash
)
// ActivePrecompiles returns the addresses of the precompiles enabled with the current
// configuration
func (evm *EVM) ActivePrecompiles() []common.Address {
switch {
case evm.chainRules.IsYoloV2:
return PrecompiledAddressesYoloV2
case evm.chainRules.IsIstanbul:
return PrecompiledAddressesIstanbul
case evm.chainRules.IsByzantium:
return PrecompiledAddressesByzantium
default:
return PrecompiledAddressesHomestead
}
}
func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) {
var precompiles map[common.Address]PrecompiledContract
switch {
case evm.chainRules.IsYoloV1:
precompiles = PrecompiledContractsYoloV1
case evm.chainRules.IsYoloV2:
precompiles = PrecompiledContractsYoloV2
case evm.chainRules.IsIstanbul:
precompiles = PrecompiledContractsIstanbul
case evm.chainRules.IsByzantium:
@ -416,7 +431,11 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64,
}
nonce := evm.StateDB.GetNonce(caller.Address())
evm.StateDB.SetNonce(caller.Address(), nonce+1)
// We add this to the access list _before_ taking a snapshot. Even if the creation fails,
// the access-list change should not be rolled back
if evm.chainRules.IsYoloV2 {
evm.StateDB.AddAddressToAccessList(address)
}
// Ensure there's no existing contract already at the designated address
contractHash := evm.StateDB.GetCodeHash(address)
if evm.StateDB.GetNonce(address) != 0 || (contractHash != (common.Hash{}) && contractHash != emptyCodeHash) {

6
core/vm/gen_structlog.go
View file

@ -24,7 +24,7 @@ func (s StructLog) MarshalJSON() ([]byte, error) {
MemorySize int `json:"memSize"`
Stack []*math.HexOrDecimal256 `json:"stack"`
ReturnStack []math.HexOrDecimal64 `json:"returnStack"`
ReturnData []byte `json:"returnData"`
ReturnData hexutil.Bytes `json:"returnData"`
Storage map[common.Hash]common.Hash `json:"-"`
Depth int `json:"depth"`
RefundCounter uint64 `json:"refund"`
@ -72,7 +72,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
MemorySize *int `json:"memSize"`
Stack []*math.HexOrDecimal256 `json:"stack"`
ReturnStack []math.HexOrDecimal64 `json:"returnStack"`
ReturnData []byte `json:"returnData"`
ReturnData *hexutil.Bytes `json:"returnData"`
Storage map[common.Hash]common.Hash `json:"-"`
Depth *int `json:"depth"`
RefundCounter *uint64 `json:"refund"`
@ -113,7 +113,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
}
}
if dec.ReturnData != nil {
s.ReturnData = dec.ReturnData
s.ReturnData = *dec.ReturnData
}
if dec.Storage != nil {
s.Storage = dec.Storage

9
core/vm/interface.go
View file

@ -57,6 +57,15 @@ type StateDB interface {
// is defined according to EIP161 (balance = nonce = code = 0).
Empty(common.Address) bool
AddressInAccessList(addr common.Address) bool
SlotInAccessList(addr common.Address, slot common.Hash) (addressOk bool, slotOk bool)
// AddAddressToAccessList adds the given address to the access list. This operation is safe to perform
// even if the feature/fork is not active yet
AddAddressToAccessList(addr common.Address)
// AddSlotToAccessList adds the given (address,slot) to the access list. This operation is safe to perform
// even if the feature/fork is not active yet
AddSlotToAccessList(addr common.Address, slot common.Hash)
RevertToSnapshot(int)
Snapshot() int

4
core/vm/interpreter.go
View file

@ -99,8 +99,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter {
if cfg.JumpTable[STOP] == nil {
var jt JumpTable
switch {
case evm.chainRules.IsYoloV1:
jt = yoloV1InstructionSet
case evm.chainRules.IsYoloV2:
jt = yoloV2InstructionSet
case evm.chainRules.IsIstanbul:
jt = istanbulInstructionSet
case evm.chainRules.IsConstantinople:

10
core/vm/jump_table.go
View file

@ -56,17 +56,19 @@ var (
byzantiumInstructionSet = newByzantiumInstructionSet()
constantinopleInstructionSet = newConstantinopleInstructionSet()
istanbulInstructionSet = newIstanbulInstructionSet()
yoloV1InstructionSet = newYoloV1InstructionSet()
yoloV2InstructionSet = newYoloV2InstructionSet()
)
// JumpTable contains the EVM opcodes supported at a given fork.
type JumpTable [256]*operation
func newYoloV1InstructionSet() JumpTable {
// newYoloV2InstructionSet creates an instructionset containing
// - "EIP-2315: Simple Subroutines"
// - "EIP-2929: Gas cost increases for state access opcodes"
func newYoloV2InstructionSet() JumpTable {
instructionSet := newIstanbulInstructionSet()
enable2315(&instructionSet) // Subroutines - https://eips.ethereum.org/EIPS/eip-2315
enable2929(&instructionSet) // Access lists for trie accesses https://eips.ethereum.org/EIPS/eip-2929
return instructionSet
}

17
core/vm/logger.go
View file

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
)
var errTraceLimitReached = errors.New("the number of logs reached the specified limit")
@ -53,6 +54,8 @@ type LogConfig struct {
DisableReturnData bool // disable return data capture
Debug bool // print output during capture end
Limit int // maximum length of output, but zero means unlimited
// Chain overrides, can be used to execute a trace using future fork rules
Overrides *params.ChainConfig `json:"overrides,omitempty"`
}
//go:generate gencodec -type StructLog -field-override structLogMarshaling -out gen_structlog.go
@ -82,6 +85,7 @@ type structLogMarshaling struct {
Gas math.HexOrDecimal64
GasCost math.HexOrDecimal64
Memory hexutil.Bytes
ReturnData hexutil.Bytes
OpName string `json:"opName"` // adds call to OpName() in MarshalJSON
ErrorString string `json:"error"` // adds call to ErrorString() in MarshalJSON
}
@ -313,8 +317,8 @@ func (t *mdLogger) CaptureStart(from common.Address, to common.Address, create b
}
fmt.Fprintf(t.out, `
| Pc | Op | Cost | Stack | RStack |
|-------|-------------|------|-----------|-----------|
| Pc | Op | Cost | Stack | RStack | Refund |
|-------|-------------|------|-----------|-----------|---------|
`)
return nil
}
@ -326,7 +330,7 @@ func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64
// format stack
var a []string
for _, elem := range stack.data {
a = append(a, fmt.Sprintf("%d", elem))
a = append(a, fmt.Sprintf("%v", elem.String()))
}
b := fmt.Sprintf("[%v]", strings.Join(a, ","))
fmt.Fprintf(t.out, "%10v |", b)
@ -339,6 +343,7 @@ func (t *mdLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64
b = fmt.Sprintf("[%v]", strings.Join(a, ","))
fmt.Fprintf(t.out, "%10v |", b)
}
fmt.Fprintf(t.out, "%10v |", env.StateDB.GetRefund())
fmt.Fprintln(t.out, "")
if err != nil {
fmt.Fprintf(t.out, "Error: %v\n", err)
@ -354,11 +359,7 @@ func (t *mdLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64
}
func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) error {
fmt.Fprintf(t.out, `
Output: 0x%x
Consumed gas: %d
Error: %v
`,
fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n",
output, gasUsed, err)
return nil
}

222
core/vm/operations_acl.go Normal file
View file

@ -0,0 +1,222 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package vm
import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/params"
)
const (
ColdAccountAccessCostEIP2929 = uint64(2600) // COLD_ACCOUNT_ACCESS_COST
ColdSloadCostEIP2929 = uint64(2100) // COLD_SLOAD_COST
WarmStorageReadCostEIP2929 = uint64(100) // WARM_STORAGE_READ_COST
)
// gasSStoreEIP2929 implements gas cost for SSTORE according to EIP-2929"
//
// When calling SSTORE, check if the (address, storage_key) pair is in accessed_storage_keys.
// If it is not, charge an additional COLD_SLOAD_COST gas, and add the pair to accessed_storage_keys.
// Additionally, modify the parameters defined in EIP 2200 as follows:
//
// Parameter Old value New value
// SLOAD_GAS 800 = WARM_STORAGE_READ_COST
// SSTORE_RESET_GAS 5000 5000 - COLD_SLOAD_COST
//
//The other parameters defined in EIP 2200 are unchanged.
// see gasSStoreEIP2200(...) in core/vm/gas_table.go for more info about how EIP 2200 is specified
func gasSStoreEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
// If we fail the minimum gas availability invariant, fail (0)
if contract.Gas <= params.SstoreSentryGasEIP2200 {
return 0, errors.New("not enough gas for reentrancy sentry")
}
// Gas sentry honoured, do the actual gas calculation based on the stored value
var (
y, x = stack.Back(1), stack.peek()
slot = common.Hash(x.Bytes32())
current = evm.StateDB.GetState(contract.Address(), slot)
cost = uint64(0)
)
// Check slot presence in the access list
if addrPresent, slotPresent := evm.StateDB.SlotInAccessList(contract.Address(), slot); !slotPresent {
cost = ColdSloadCostEIP2929
// If the caller cannot afford the cost, this change will be rolled back
evm.StateDB.AddSlotToAccessList(contract.Address(), slot)
if !addrPresent {
// Once we're done with YOLOv2 and schedule this for mainnet, might
// be good to remove this panic here, which is just really a
// canary to have during testing
panic("impossible case: address was not present in access list during sstore op")
}
}
value := common.Hash(y.Bytes32())
if current == value { // noop (1)
// EIP 2200 original clause:
// return params.SloadGasEIP2200, nil
return cost + WarmStorageReadCostEIP2929, nil // SLOAD_GAS
}
original := evm.StateDB.GetCommittedState(contract.Address(), common.Hash(x.Bytes32()))
if original == current {
if original == (common.Hash{}) { // create slot (2.1.1)
return cost + params.SstoreSetGasEIP2200, nil
}
if value == (common.Hash{}) { // delete slot (2.1.2b)
evm.StateDB.AddRefund(params.SstoreClearsScheduleRefundEIP2200)
}
// EIP-2200 original clause:
// return params.SstoreResetGasEIP2200, nil // write existing slot (2.1.2)
return cost + (params.SstoreResetGasEIP2200 - ColdSloadCostEIP2929), nil // write existing slot (2.1.2)
}
if original != (common.Hash{}) {
if current == (common.Hash{}) { // recreate slot (2.2.1.1)
evm.StateDB.SubRefund(params.SstoreClearsScheduleRefundEIP2200)
} else if value == (common.Hash{}) { // delete slot (2.2.1.2)
evm.StateDB.AddRefund(params.SstoreClearsScheduleRefundEIP2200)
}
}
if original == value {
if original == (common.Hash{}) { // reset to original inexistent slot (2.2.2.1)
// EIP 2200 Original clause:
//evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - params.SloadGasEIP2200)
evm.StateDB.AddRefund(params.SstoreSetGasEIP2200 - WarmStorageReadCostEIP2929)
} else { // reset to original existing slot (2.2.2.2)
// EIP 2200 Original clause:
// evm.StateDB.AddRefund(params.SstoreResetGasEIP2200 - params.SloadGasEIP2200)
// - SSTORE_RESET_GAS redefined as (5000 - COLD_SLOAD_COST)
// - SLOAD_GAS redefined as WARM_STORAGE_READ_COST
// Final: (5000 - COLD_SLOAD_COST) - WARM_STORAGE_READ_COST
evm.StateDB.AddRefund((params.SstoreResetGasEIP2200 - ColdSloadCostEIP2929) - WarmStorageReadCostEIP2929)
}
}
// EIP-2200 original clause:
//return params.SloadGasEIP2200, nil // dirty update (2.2)
return cost + WarmStorageReadCostEIP2929, nil // dirty update (2.2)
}
// gasSLoadEIP2929 calculates dynamic gas for SLOAD according to EIP-2929
// For SLOAD, if the (address, storage_key) pair (where address is the address of the contract
// whose storage is being read) is not yet in accessed_storage_keys,
// charge 2100 gas and add the pair to accessed_storage_keys.
// If the pair is already in accessed_storage_keys, charge 100 gas.
func gasSLoadEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
loc := stack.peek()
slot := common.Hash(loc.Bytes32())
// Check slot presence in the access list
if _, slotPresent := evm.StateDB.SlotInAccessList(contract.Address(), slot); !slotPresent {
// If the caller cannot afford the cost, this change will be rolled back
// If he does afford it, we can skip checking the same thing later on, during execution
evm.StateDB.AddSlotToAccessList(contract.Address(), slot)
return ColdSloadCostEIP2929, nil
}
return WarmStorageReadCostEIP2929, nil
}
// gasExtCodeCopyEIP2929 implements extcodecopy according to EIP-2929
// EIP spec:
// > If the target is not in accessed_addresses,
// > charge COLD_ACCOUNT_ACCESS_COST gas, and add the address to accessed_addresses.
// > Otherwise, charge WARM_STORAGE_READ_COST gas.
func gasExtCodeCopyEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
// memory expansion first (dynamic part of pre-2929 implementation)
gas, err := gasExtCodeCopy(evm, contract, stack, mem, memorySize)
if err != nil {
return 0, err
}
addr := common.Address(stack.peek().Bytes20())
// Check slot presence in the access list
if !evm.StateDB.AddressInAccessList(addr) {
evm.StateDB.AddAddressToAccessList(addr)
var overflow bool
// We charge (cold-warm), since 'warm' is already charged as constantGas
if gas, overflow = math.SafeAdd(gas, ColdAccountAccessCostEIP2929-WarmStorageReadCostEIP2929); overflow {
return 0, ErrGasUintOverflow
}
return gas, nil
}
return gas, nil
}
// gasEip2929AccountCheck checks whether the first stack item (as address) is present in the access list.
// If it is, this method returns '0', otherwise 'cold-warm' gas, presuming that the opcode using it
// is also using 'warm' as constant factor.
// This method is used by:
// - extcodehash,
// - extcodesize,
// - (ext) balance
func gasEip2929AccountCheck(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
addr := common.Address(stack.peek().Bytes20())
// Check slot presence in the access list
if !evm.StateDB.AddressInAccessList(addr) {
// If the caller cannot afford the cost, this change will be rolled back
evm.StateDB.AddAddressToAccessList(addr)
// The warm storage read cost is already charged as constantGas
return ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929, nil
}
return 0, nil
}
func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc {
return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
addr := common.Address(stack.Back(1).Bytes20())
// Check slot presence in the access list
if !evm.StateDB.AddressInAccessList(addr) {
evm.StateDB.AddAddressToAccessList(addr)
// The WarmStorageReadCostEIP2929 (100) is already deducted in the form of a constant cost
if !contract.UseGas(ColdAccountAccessCostEIP2929 - WarmStorageReadCostEIP2929) {
return 0, ErrOutOfGas
}
}
// Now call the old calculator, which takes into account
// - create new account
// - transfer value
// - memory expansion
// - 63/64ths rule
return oldCalculator(evm, contract, stack, mem, memorySize)
}
}
var (
gasCallEIP2929 = makeCallVariantGasCallEIP2929(gasCall)
gasDelegateCallEIP2929 = makeCallVariantGasCallEIP2929(gasDelegateCall)
gasStaticCallEIP2929 = makeCallVariantGasCallEIP2929(gasStaticCall)
gasCallCodeEIP2929 = makeCallVariantGasCallEIP2929(gasCallCode)
)
func gasSelfdestructEIP2929(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) {
var (
gas uint64
address = common.Address(stack.peek().Bytes20())
)
if !evm.StateDB.AddressInAccessList(address) {
// If the caller cannot afford the cost, this change will be rolled back
evm.StateDB.AddAddressToAccessList(address)
gas = ColdAccountAccessCostEIP2929
}
// if empty and transfers value
if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 {
gas += params.CreateBySelfdestructGas
}
if !evm.StateDB.HasSuicided(contract.Address()) {
evm.StateDB.AddRefund(params.SelfdestructRefundGas)
}
return gas, nil
}

24
core/vm/runtime/runtime.go
View file

@ -65,7 +65,7 @@ func setDefaults(cfg *Config) {
PetersburgBlock: new(big.Int),
IstanbulBlock: new(big.Int),
MuirGlacierBlock: new(big.Int),
YoloV1Block: nil,
YoloV2Block: nil,
}
}
@ -113,6 +113,14 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
vmenv = NewEnv(cfg)
sender = vm.AccountRef(cfg.Origin)
)
if cfg.ChainConfig.IsYoloV2(vmenv.BlockNumber) {
cfg.State.AddAddressToAccessList(cfg.Origin)
cfg.State.AddAddressToAccessList(address)
for _, addr := range vmenv.ActivePrecompiles() {
cfg.State.AddAddressToAccessList(addr)
cfg.State.AddAddressToAccessList(addr)
}
}
cfg.State.CreateAccount(address)
// set the receiver's (the executing contract) code for execution.
cfg.State.SetCode(address, code)
@ -142,6 +150,12 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) {
vmenv = NewEnv(cfg)
sender = vm.AccountRef(cfg.Origin)
)
if cfg.ChainConfig.IsYoloV2(vmenv.BlockNumber) {
cfg.State.AddAddressToAccessList(cfg.Origin)
for _, addr := range vmenv.ActivePrecompiles() {
cfg.State.AddAddressToAccessList(addr)
}
}
// Call the code with the given configuration.
code, address, leftOverGas, err := vmenv.Create(
@ -164,6 +178,14 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er
vmenv := NewEnv(cfg)
sender := cfg.State.GetOrNewStateObject(cfg.Origin)
if cfg.ChainConfig.IsYoloV2(vmenv.BlockNumber) {
cfg.State.AddAddressToAccessList(cfg.Origin)
cfg.State.AddAddressToAccessList(address)
for _, addr := range vmenv.ActivePrecompiles() {
cfg.State.AddAddressToAccessList(addr)
}
}
// Call the code with the given configuration.
ret, leftOverGas, err := vmenv.Call(
sender,

112
core/vm/runtime/runtime_test.go
View file

@ -722,3 +722,115 @@ func BenchmarkSimpleLoop(b *testing.B) {
//benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b)
//benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b)
}
// TestEip2929Cases contains various testcases that are used for
// EIP-2929 about gas repricings
func TestEip2929Cases(t *testing.T) {
id := 1
prettyPrint := func(comment string, code []byte) {
instrs := make([]string, 0)
it := asm.NewInstructionIterator(code)
for it.Next() {
if it.Arg() != nil && 0 < len(it.Arg()) {
instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg()))
} else {
instrs = append(instrs, fmt.Sprintf("%v", it.Op()))
}
}
ops := strings.Join(instrs, ", ")
fmt.Printf("### Case %d\n\n", id)
id++
fmt.Printf("%v\n\nBytecode: \n```\n0x%x\n```\nOperations: \n```\n%v\n```\n\n",
comment,
code, ops)
Execute(code, nil, &Config{
EVMConfig: vm.Config{
Debug: true,
Tracer: vm.NewMarkdownLogger(nil, os.Stdout),
ExtraEips: []int{2929},
},
})
}
{ // First eip testcase
code := []byte{
// Three checks against a precompile
byte(vm.PUSH1), 1, byte(vm.EXTCODEHASH), byte(vm.POP),
byte(vm.PUSH1), 2, byte(vm.EXTCODESIZE), byte(vm.POP),
byte(vm.PUSH1), 3, byte(vm.BALANCE), byte(vm.POP),
// Three checks against a non-precompile
byte(vm.PUSH1), 0xf1, byte(vm.EXTCODEHASH), byte(vm.POP),
byte(vm.PUSH1), 0xf2, byte(vm.EXTCODESIZE), byte(vm.POP),
byte(vm.PUSH1), 0xf3, byte(vm.BALANCE), byte(vm.POP),
// Same three checks (should be cheaper)
byte(vm.PUSH1), 0xf2, byte(vm.EXTCODEHASH), byte(vm.POP),
byte(vm.PUSH1), 0xf3, byte(vm.EXTCODESIZE), byte(vm.POP),
byte(vm.PUSH1), 0xf1, byte(vm.BALANCE), byte(vm.POP),
// Check the origin, and the 'this'
byte(vm.ORIGIN), byte(vm.BALANCE), byte(vm.POP),
byte(vm.ADDRESS), byte(vm.BALANCE), byte(vm.POP),
byte(vm.STOP),
}
prettyPrint("This checks `EXT`(codehash,codesize,balance) of precompiles, which should be `100`, "+
"and later checks the same operations twice against some non-precompiles. "+
"Those are cheaper second time they are accessed. Lastly, it checks the `BALANCE` of `origin` and `this`.", code)
}
{ // EXTCODECOPY
code := []byte{
// extcodecopy( 0xff,0,0,0,0)
byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, //length, codeoffset, memoffset
byte(vm.PUSH1), 0xff, byte(vm.EXTCODECOPY),
// extcodecopy( 0xff,0,0,0,0)
byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, //length, codeoffset, memoffset
byte(vm.PUSH1), 0xff, byte(vm.EXTCODECOPY),
// extcodecopy( this,0,0,0,0)
byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, byte(vm.PUSH1), 0x00, //length, codeoffset, memoffset
byte(vm.ADDRESS), byte(vm.EXTCODECOPY),
byte(vm.STOP),
}
prettyPrint("This checks `extcodecopy( 0xff,0,0,0,0)` twice, (should be expensive first time), "+
"and then does `extcodecopy( this,0,0,0,0)`.", code)
}
{ // SLOAD + SSTORE
code := []byte{
// Add slot `0x1` to access list
byte(vm.PUSH1), 0x01, byte(vm.SLOAD), byte(vm.POP), // SLOAD( 0x1) (add to access list)
// Write to `0x1` which is already in access list
byte(vm.PUSH1), 0x11, byte(vm.PUSH1), 0x01, byte(vm.SSTORE), // SSTORE( loc: 0x01, val: 0x11)
// Write to `0x2` which is not in access list
byte(vm.PUSH1), 0x11, byte(vm.PUSH1), 0x02, byte(vm.SSTORE), // SSTORE( loc: 0x02, val: 0x11)
// Write again to `0x2`
byte(vm.PUSH1), 0x11, byte(vm.PUSH1), 0x02, byte(vm.SSTORE), // SSTORE( loc: 0x02, val: 0x11)
// Read slot in access list (0x2)
byte(vm.PUSH1), 0x02, byte(vm.SLOAD), // SLOAD( 0x2)
// Read slot in access list (0x1)
byte(vm.PUSH1), 0x01, byte(vm.SLOAD), // SLOAD( 0x1)
}
prettyPrint("This checks `sload( 0x1)` followed by `sstore(loc: 0x01, val:0x11)`, then 'naked' sstore:"+
"`sstore(loc: 0x02, val:0x11)` twice, and `sload(0x2)`, `sload(0x1)`. ", code)
}
{ // Call variants
code := []byte{
// identity precompile
byte(vm.PUSH1), 0x0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1),
byte(vm.PUSH1), 0x04, byte(vm.PUSH1), 0x0, byte(vm.CALL), byte(vm.POP),
// random account - call 1
byte(vm.PUSH1), 0x0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1),
byte(vm.PUSH1), 0xff, byte(vm.PUSH1), 0x0, byte(vm.CALL), byte(vm.POP),
// random account - call 2
byte(vm.PUSH1), 0x0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1),
byte(vm.PUSH1), 0xff, byte(vm.PUSH1), 0x0, byte(vm.STATICCALL), byte(vm.POP),
}
prettyPrint("This calls the `identity`-precompile (cheap), then calls an account (expensive) and `staticcall`s the same"+
"account (cheap)", code)
}
}

2
eth/api.go
View file

@ -389,6 +389,8 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta
if err != nil {
return state.IteratorDump{}, err
}
} else {
return state.IteratorDump{}, errors.New("either block number or block hash must be specified")
}
if maxResults > AccountRangeMaxResults || maxResults <= 0 {

30
eth/api_tracer.go
View file

@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/eth/tracers"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
@ -561,9 +562,28 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
// Execute transaction, either tracing all or just the requested one
var (
signer = types.MakeSigner(api.eth.blockchain.Config(), block.Number())
dumps []string
signer = types.MakeSigner(api.eth.blockchain.Config(), block.Number())
dumps []string
chainConfig = api.eth.blockchain.Config()
canon = true
)
// Check if there are any overrides: the caller may wish to enable a future
// fork when executing this block. Note, such overrides are only applicable to the
// actual specified block, not any preceding blocks that we have to go through
// in order to obtain the state.
// Therefore, it's perfectly valid to specify `"futureForkBlock": 0`, to enable `futureFork`
if config != nil && config.Overrides != nil {
// Copy the config, to not screw up the main config
// Note: the Clique-part is _not_ deep copied
chainConfigCopy := new(params.ChainConfig)
*chainConfigCopy = *chainConfig
chainConfig = chainConfigCopy
if yolov2 := config.Overrides.YoloV2Block; yolov2 != nil {
chainConfig.YoloV2Block = yolov2
canon = false
}
}
for i, tx := range block.Transactions() {
// Prepare the trasaction for un-traced execution
var (
@ -579,7 +599,9 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
if tx.Hash() == txHash || txHash == (common.Hash{}) {
// Generate a unique temporary file to dump it into
prefix := fmt.Sprintf("block_%#x-%d-%#x-", block.Hash().Bytes()[:4], i, tx.Hash().Bytes()[:4])
if !canon {
prefix = fmt.Sprintf("%valt-", prefix)
}
dump, err = ioutil.TempFile(os.TempDir(), prefix)
if err != nil {
return nil, err
@ -595,7 +617,7 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block
}
}
// Execute the transaction and flush any traces to disk
vmenv := vm.NewEVM(vmctx, statedb, api.eth.blockchain.Config(), vmConf)
vmenv := vm.NewEVM(vmctx, statedb, chainConfig, vmConf)
_, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas()))
if writer != nil {
writer.Flush()

94
internal/build/archive.go
View file

@ -184,24 +184,35 @@ func (a *TarballArchive) Close() error {
return a.file.Close()
}
func ExtractTarballArchive(archive string, dest string) error {
// We're only interested in gzipped archives, wrap the reader now
// ExtractArchive unpacks a .zip or .tar.gz archive to the destination directory.
func ExtractArchive(archive string, dest string) error {
ar, err := os.Open(archive)
if err != nil {
return err
}
defer ar.Close()
switch {
case strings.HasSuffix(archive, ".tar.gz"):
return extractTarball(ar, dest)
case strings.HasSuffix(archive, ".zip"):
return extractZip(ar, dest)
default:
return fmt.Errorf("unhandled archive type %s", archive)
}
}
// extractTarball unpacks a .tar.gz file.
func extractTarball(ar io.Reader, dest string) error {
gzr, err := gzip.NewReader(ar)
if err != nil {
return err
}
defer gzr.Close()
// Iterate over all the files in the tarball
tr := tar.NewReader(gzr)
for {
// Fetch the next tarball header and abort if needed
// Move to the next file header.
header, err := tr.Next()
if err != nil {
if err == io.EOF {
@ -209,22 +220,69 @@ func ExtractTarballArchive(archive string, dest string) error {
}
return err
}
// Figure out the target and create it
target := filepath.Join(dest, header.Name)
switch header.Typeflag {
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
// We only care about regular files, directory modes
// and special file types are not supported.
if header.Typeflag == tar.TypeReg {
armode := header.FileInfo().Mode()
err := extractFile(header.Name, armode, tr, dest)
if err != nil {
return err
return fmt.Errorf("extract %s: %v", header.Name, err)
}
if _, err := io.Copy(file, tr); err != nil {
return err
}
file.Close()
}
}
}
// extractZip unpacks the given .zip file.
func extractZip(ar *os.File, dest string) error {
info, err := ar.Stat()
if err != nil {
return err
}
zr, err := zip.NewReader(ar, info.Size())
if err != nil {
return err
}
for _, zf := range zr.File {
if !zf.Mode().IsRegular() {
continue
}
data, err := zf.Open()
if err != nil {
return err
}
err = extractFile(zf.Name, zf.Mode(), data, dest)
data.Close()
if err != nil {
return fmt.Errorf("extract %s: %v", zf.Name, err)
}
}
return nil
}
// extractFile extracts a single file from an archive.
func extractFile(arpath string, armode os.FileMode, data io.Reader, dest string) error {
// Check that path is inside destination directory.
target := filepath.Join(dest, filepath.FromSlash(arpath))
if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) {
return fmt.Errorf("path %q escapes archive destination", target)
}
// Ensure the destination directory exists.
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
return err
}
// Copy file data.
file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, armode)
if err != nil {
return err
}
if _, err := io.Copy(file, data); err != nil {
file.Close()
os.Remove(target)
return err
}
return file.Close()
}

27
internal/build/util.go
View file

@ -20,6 +20,8 @@ import (
"bytes"
"flag"
"fmt"
"go/parser"
"go/token"
"io"
"io/ioutil"
"log"
@ -152,3 +154,28 @@ func UploadSFTP(identityFile, host, dir string, files []string) error {
stdin.Close()
return sftp.Wait()
}
// FindMainPackages finds all 'main' packages in the given directory and returns their
// package paths.
func FindMainPackages(dir string) []string {
var commands []string
cmds, err := ioutil.ReadDir(dir)
if err != nil {
log.Fatal(err)
}
for _, cmd := range cmds {
pkgdir := filepath.Join(dir, cmd.Name())
pkgs, err := parser.ParseDir(token.NewFileSet(), pkgdir, nil, parser.PackageClauseOnly)
if err != nil {
log.Fatal(err)
}
for name := range pkgs {
if name == "main" {
path := "./" + filepath.ToSlash(pkgdir)
commands = append(commands, path)
break
}
}
}
return commands
}

13
internal/ethapi/api.go
View file

@ -599,7 +599,7 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre
if storageError != nil {
return nil, storageError
}
storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)}
storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), toHexSlice(proof)}
} else {
storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}}
}
@ -613,7 +613,7 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre
return &AccountResult{
Address: address,
AccountProof: common.ToHexArray(accountProof),
AccountProof: toHexSlice(accountProof),
Balance: (*hexutil.Big)(state.GetBalance(address)),
CodeHash: codeHash,
Nonce: hexutil.Uint64(state.GetNonce(address)),
@ -1943,3 +1943,12 @@ func checkTxFee(gasPrice *big.Int, gas uint64, cap float64) error {
}
return nil
}
// toHexSlice creates a slice of hex-strings based on []byte.
func toHexSlice(b [][]byte) []string {
r := make([]string, len(b))
for i := range b {
r[i] = hexutil.Encode(b[i])
}
return r
}

167
internal/utesting/utesting.go
View file

@ -25,6 +25,7 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"regexp"
"runtime"
"sync"
@ -63,32 +64,165 @@ func MatchTests(tests []Test, expr string) []Test {
// RunTests executes all given tests in order and returns their results.
// If the report writer is non-nil, a test report is written to it in real time.
func RunTests(tests []Test, report io.Writer) []Result {
results := make([]Result, len(tests))
if report == nil {
report = ioutil.Discard
}
results := run(tests, newConsoleOutput(report))
fails := CountFailures(results)
fmt.Fprintf(report, "%v/%v tests passed.\n", len(tests)-fails, len(tests))
return results
}
// RunTAP runs the given tests and writes Test Anything Protocol output
// to the report writer.
func RunTAP(tests []Test, report io.Writer) []Result {
return run(tests, newTAP(report, len(tests)))
}
func run(tests []Test, output testOutput) []Result {
var results = make([]Result, len(tests))
for i, test := range tests {
var output io.Writer
buffer := new(bytes.Buffer)
output = buffer
if report != nil {
output = io.MultiWriter(buffer, report)
}
logOutput := io.MultiWriter(buffer, output)
output.testStart(test.Name)
start := time.Now()
results[i].Name = test.Name
results[i].Failed = run(test, output)
results[i].Failed = runTest(test, logOutput)
results[i].Duration = time.Since(start)
results[i].Output = buffer.String()
if report != nil {
printResult(results[i], report)
}
output.testResult(results[i])
}
return results
}
func printResult(r Result, w io.Writer) {
// testOutput is implemented by output formats.
type testOutput interface {
testStart(name string)
Write([]byte) (int, error)
testResult(Result)
}
// consoleOutput prints test results similarly to go test.
type consoleOutput struct {
out io.Writer
indented *indentWriter
curTest string
wroteHeader bool
}
func newConsoleOutput(w io.Writer) *consoleOutput {
return &consoleOutput{
out: w,
indented: newIndentWriter(" ", w),
}
}
// testStart signals the start of a new test.
func (c *consoleOutput) testStart(name string) {
c.curTest = name
c.wroteHeader = false
}
// Write handles test log output.
func (c *consoleOutput) Write(b []byte) (int, error) {
if !c.wroteHeader {
// This is the first output line from the test. Print a "-- RUN" header.
fmt.Fprintln(c.out, "-- RUN", c.curTest)
c.wroteHeader = true
}
return c.indented.Write(b)
}
// testResult prints the final test result line.
func (c *consoleOutput) testResult(r Result) {
c.indented.flush()
pd := r.Duration.Truncate(100 * time.Microsecond)
if r.Failed {
fmt.Fprintf(w, "-- FAIL %s (%v)\n", r.Name, pd)
fmt.Fprintf(c.out, "-- FAIL %s (%v)\n", r.Name, pd)
} else {
fmt.Fprintf(w, "-- OK %s (%v)\n", r.Name, pd)
fmt.Fprintf(c.out, "-- OK %s (%v)\n", r.Name, pd)
}
}
// tapOutput produces Test Anything Protocol v13 output.
type tapOutput struct {
out io.Writer
indented *indentWriter
counter int
}
func newTAP(out io.Writer, numTests int) *tapOutput {
fmt.Fprintf(out, "1..%d\n", numTests)
return &tapOutput{
out: out,
indented: newIndentWriter("# ", out),
}
}
func (t *tapOutput) testStart(name string) {
t.counter++
}
// Write does nothing for TAP because there is no real-time output of test logs.
func (t *tapOutput) Write(b []byte) (int, error) {
return len(b), nil
}
func (t *tapOutput) testResult(r Result) {
status := "ok"
if r.Failed {
status = "not ok"
}
fmt.Fprintln(t.out, status, t.counter, r.Name)
t.indented.Write([]byte(r.Output))
t.indented.flush()
}
// indentWriter indents all written text.
type indentWriter struct {
out io.Writer
indent string
inLine bool
}
func newIndentWriter(indent string, out io.Writer) *indentWriter {
return &indentWriter{out: out, indent: indent}
}
func (w *indentWriter) Write(b []byte) (n int, err error) {
for len(b) > 0 {
if !w.inLine {
if _, err = io.WriteString(w.out, w.indent); err != nil {
return n, err
}
w.inLine = true
}
end := bytes.IndexByte(b, '\n')
if end == -1 {
nn, err := w.out.Write(b)
n += nn
return n, err
}
line := b[:end+1]
nn, err := w.out.Write(line)
n += nn
if err != nil {
return n, err
}
b = b[end+1:]
w.inLine = false
}
return n, err
}
// flush ensures the current line is terminated.
func (w *indentWriter) flush() {
if w.inLine {
fmt.Println(w.out)
w.inLine = false
}
}
@ -106,11 +240,11 @@ func CountFailures(rr []Result) int {
// Run executes a single test.
func Run(test Test) (bool, string) {
output := new(bytes.Buffer)
failed := run(test, output)
failed := runTest(test, output)
return failed, output.String()
}
func run(test Test, output io.Writer) bool {
func runTest(test Test, output io.Writer) bool {
t := &T{output: output}
done := make(chan struct{})
go func() {
@ -137,6 +271,9 @@ type T struct {
output io.Writer
}
// Helper exists for compatibility with testing.T.
func (t *T) Helper() {}
// FailNow marks the test as having failed and stops its execution by calling
// runtime.Goexit (which then runs all deferred calls in the current goroutine).
func (t *T) FailNow() {

84
internal/utesting/utesting_test.go
View file

@ -17,6 +17,8 @@
package utesting
import (
"bytes"
"regexp"
"strings"
"testing"
)
@ -53,3 +55,85 @@ func TestTest(t *testing.T) {
t.Fatalf("wrong result for panicking test: %#v", results[2])
}
}
var outputTests = []Test{
{
Name: "TestWithLogs",
Fn: func(t *T) {
t.Log("output line 1")
t.Log("output line 2\noutput line 3")
},
},
{
Name: "TestNoLogs",
Fn: func(t *T) {},
},
{
Name: "FailWithLogs",
Fn: func(t *T) {
t.Log("output line 1")
t.Error("failed 1")
},
},
{
Name: "FailMessage",
Fn: func(t *T) {
t.Error("failed 2")
},
},
{
Name: "FailNoOutput",
Fn: func(t *T) {
t.Fail()
},
},
}
func TestOutput(t *testing.T) {
var buf bytes.Buffer
RunTests(outputTests, &buf)
want := regexp.MustCompile(`
^-- RUN TestWithLogs
output line 1
output line 2
output line 3
-- OK TestWithLogs \([^)]+\)
-- OK TestNoLogs \([^)]+\)
-- RUN FailWithLogs
output line 1
failed 1
-- FAIL FailWithLogs \([^)]+\)
-- RUN FailMessage
failed 2
-- FAIL FailMessage \([^)]+\)
-- FAIL FailNoOutput \([^)]+\)
2/5 tests passed.
$`[1:])
if !want.MatchString(buf.String()) {
t.Fatalf("output does not match: %q", buf.String())
}
}
func TestOutputTAP(t *testing.T) {
var buf bytes.Buffer
RunTAP(outputTests, &buf)
want := `
1..5
ok 1 TestWithLogs
# output line 1
# output line 2
# output line 3
ok 2 TestNoLogs
not ok 3 FailWithLogs
# output line 1
# failed 1
not ok 4 FailMessage
# failed 2
not ok 5 FailNoOutput
`
if buf.String() != want[1:] {
t.Fatalf("output does not match: %q", buf.String())
}
}

8
les/client_handler.go
View file

@ -102,13 +102,7 @@ func (h *clientHandler) handle(p *serverPeer) error {
p.Log().Debug("Light Ethereum peer connected", "name", p.Name())
// Execute the LES handshake
var (
head = h.backend.blockchain.CurrentHeader()
hash = head.Hash()
number = head.Number.Uint64()
td = h.backend.blockchain.GetTd(hash, number)
)
if err := p.Handshake(td, hash, number, h.backend.blockchain.Genesis().Hash(), nil); err != nil {
if err := p.Handshake(h.backend.blockchain.Genesis().Hash()); err != nil {
p.Log().Debug("Light Ethereum handshake failed", "err", err)
return err
}

49
les/clientpool.go
View file

@ -18,7 +18,6 @@ package les
import (
"fmt"
"reflect"
"sync"
"time"
@ -46,19 +45,6 @@ const (
inactiveTimeout = time.Second * 10
)
var (
clientPoolSetup = &nodestate.Setup{}
clientField = clientPoolSetup.NewField("clientInfo", reflect.TypeOf(&clientInfo{}))
connAddressField = clientPoolSetup.NewField("connAddr", reflect.TypeOf(""))
balanceTrackerSetup = lps.NewBalanceTrackerSetup(clientPoolSetup)
priorityPoolSetup = lps.NewPriorityPoolSetup(clientPoolSetup)
)
func init() {
balanceTrackerSetup.Connect(connAddressField, priorityPoolSetup.CapacityField)
priorityPoolSetup.Connect(balanceTrackerSetup.BalanceField, balanceTrackerSetup.UpdateFlag) // NodeBalance implements nodePriority
}
// clientPool implements a client database that assigns a priority to each client
// based on a positive and negative balance. Positive balance is externally assigned
// to prioritized clients and is decreased with connection time and processed
@ -119,8 +105,7 @@ type clientInfo struct {
}
// newClientPool creates a new client pool
func newClientPool(lespayDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID)) *clientPool {
ns := nodestate.NewNodeStateMachine(nil, nil, clock, clientPoolSetup)
func newClientPool(ns *nodestate.NodeStateMachine, lespayDb ethdb.Database, minCap uint64, connectedBias time.Duration, clock mclock.Clock, removePeer func(enode.ID)) *clientPool {
pool := &clientPool{
ns: ns,
BalanceTrackerSetup: balanceTrackerSetup,
@ -147,7 +132,7 @@ func newClientPool(lespayDb ethdb.Database, minCap uint64, connectedBias time.Du
})
ns.SubscribeState(pool.ActiveFlag.Or(pool.PriorityFlag), func(node *enode.Node, oldState, newState nodestate.Flags) {
c, _ := ns.GetField(node, clientField).(*clientInfo)
c, _ := ns.GetField(node, clientInfoField).(*clientInfo)
if c == nil {
return
}
@ -172,7 +157,7 @@ func newClientPool(lespayDb ethdb.Database, minCap uint64, connectedBias time.Du
if oldState.Equals(pool.ActiveFlag) && newState.Equals(pool.InactiveFlag) {
clientDeactivatedMeter.Mark(1)
log.Debug("Client deactivated", "id", node.ID())
c, _ := ns.GetField(node, clientField).(*clientInfo)
c, _ := ns.GetField(node, clientInfoField).(*clientInfo)
if c == nil || !c.peer.allowInactive() {
pool.removePeer(node.ID())
}
@ -190,13 +175,11 @@ func newClientPool(lespayDb ethdb.Database, minCap uint64, connectedBias time.Du
newCap, _ := newValue.(uint64)
totalConnected += newCap - oldCap
totalConnectedGauge.Update(int64(totalConnected))
c, _ := ns.GetField(node, clientField).(*clientInfo)
c, _ := ns.GetField(node, clientInfoField).(*clientInfo)
if c != nil {
c.peer.updateCapacity(newCap)
}
})
ns.Start()
return pool
}
@ -210,7 +193,6 @@ func (f *clientPool) stop() {
f.disconnectNode(node)
})
f.bt.Stop()
f.ns.Stop()
}
// connect should be called after a successful handshake. If the connection was
@ -225,7 +207,7 @@ func (f *clientPool) connect(peer clientPoolPeer) (uint64, error) {
}
// Dedup connected peers.
node, freeID := peer.Node(), peer.freeClientId()
if f.ns.GetField(node, clientField) != nil {
if f.ns.GetField(node, clientInfoField) != nil {
log.Debug("Client already connected", "address", freeID, "id", node.ID().String())
return 0, fmt.Errorf("Client already connected address=%s id=%s", freeID, node.ID().String())
}
@ -237,7 +219,7 @@ func (f *clientPool) connect(peer clientPoolPeer) (uint64, error) {
connected: true,
connectedAt: now,
}
f.ns.SetField(node, clientField, c)
f.ns.SetField(node, clientInfoField, c)
f.ns.SetField(node, connAddressField, freeID)
if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance == nil {
f.disconnect(peer)
@ -280,7 +262,7 @@ func (f *clientPool) disconnect(p clientPoolPeer) {
// disconnectNode removes node fields and flags related to connected status
func (f *clientPool) disconnectNode(node *enode.Node) {
f.ns.SetField(node, connAddressField, nil)
f.ns.SetField(node, clientField, nil)
f.ns.SetField(node, clientInfoField, nil)
}
// setDefaultFactors sets the default price factors applied to subsequently connected clients
@ -299,7 +281,8 @@ func (f *clientPool) capacityInfo() (uint64, uint64, uint64) {
defer f.lock.Unlock()
// total priority active cap will be supported when the token issuer module is added
return f.capLimit, f.pp.ActiveCapacity(), 0
_, activeCap := f.pp.Active()
return f.capLimit, activeCap, 0
}
// setLimits sets the maximum number and total capacity of connected clients,
@ -314,13 +297,13 @@ func (f *clientPool) setLimits(totalConn int, totalCap uint64) {
// setCapacity sets the assigned capacity of a connected client
func (f *clientPool) setCapacity(node *enode.Node, freeID string, capacity uint64, bias time.Duration, setCap bool) (uint64, error) {
c, _ := f.ns.GetField(node, clientField).(*clientInfo)
c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo)
if c == nil {
if setCap {
return 0, fmt.Errorf("client %064x is not connected", node.ID())
}
c = &clientInfo{node: node}
f.ns.SetField(node, clientField, c)
f.ns.SetField(node, clientInfoField, c)
f.ns.SetField(node, connAddressField, freeID)
if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance == nil {
log.Error("BalanceField is missing", "node", node.ID())
@ -328,7 +311,7 @@ func (f *clientPool) setCapacity(node *enode.Node, freeID string, capacity uint6
}
defer func() {
f.ns.SetField(node, connAddressField, nil)
f.ns.SetField(node, clientField, nil)
f.ns.SetField(node, clientInfoField, nil)
}()
}
var (
@ -370,7 +353,7 @@ func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) {
if len(ids) == 0 {
f.ns.ForEach(nodestate.Flags{}, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) {
c, _ := f.ns.GetField(node, clientField).(*clientInfo)
c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo)
if c != nil {
cb(c)
}
@ -381,12 +364,12 @@ func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) {
if node == nil {
node = enode.SignNull(&enr.Record{}, id)
}
c, _ := f.ns.GetField(node, clientField).(*clientInfo)
c, _ := f.ns.GetField(node, clientInfoField).(*clientInfo)
if c != nil {
cb(c)
} else {
c = &clientInfo{node: node}
f.ns.SetField(node, clientField, c)
f.ns.SetField(node, clientInfoField, c)
f.ns.SetField(node, connAddressField, "")
if c.balance, _ = f.ns.GetField(node, f.BalanceField).(*lps.NodeBalance); c.balance != nil {
cb(c)
@ -394,7 +377,7 @@ func (f *clientPool) forClients(ids []enode.ID, cb func(client *clientInfo)) {
log.Error("BalanceField is missing")
}
f.ns.SetField(node, connAddressField, nil)
f.ns.SetField(node, clientField, nil)
f.ns.SetField(node, clientInfoField, nil)
}
}
}

43
les/clientpool_test.go
View file

@ -64,6 +64,11 @@ type poolTestPeer struct {
inactiveAllowed bool
}
func testStateMachine() *nodestate.NodeStateMachine {
return nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup)
}
func newPoolTestPeer(i int, disconnCh chan int) *poolTestPeer {
return &poolTestPeer{
index: i,
@ -91,7 +96,7 @@ func (i *poolTestPeer) allowInactive() bool {
}
func getBalance(pool *clientPool, p *poolTestPeer) (pos, neg uint64) {
temp := pool.ns.GetField(p.node, clientField) == nil
temp := pool.ns.GetField(p.node, clientInfoField) == nil
if temp {
pool.ns.SetField(p.node, connAddressField, p.freeClientId())
}
@ -128,8 +133,9 @@ func testClientPool(t *testing.T, activeLimit, clientCount, paidCount int, rando
disconnFn = func(id enode.ID) {
disconnCh <- int(id[0]) + int(id[1])<<8
}
pool = newClientPool(db, 1, 0, &clock, disconnFn)
pool = newClientPool(testStateMachine(), db, 1, 0, &clock, disconnFn)
)
pool.ns.Start()
pool.setLimits(activeLimit, uint64(activeLimit))
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -233,7 +239,8 @@ func TestConnectPaidClient(t *testing.T) {
clock mclock.Simulated
db = rawdb.NewMemoryDatabase()
)
pool := newClientPool(db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10))
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -248,7 +255,8 @@ func TestConnectPaidClientToSmallPool(t *testing.T) {
clock mclock.Simulated
db = rawdb.NewMemoryDatabase()
)
pool := newClientPool(db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -266,7 +274,8 @@ func TestConnectPaidClientToFullPool(t *testing.T) {
db = rawdb.NewMemoryDatabase()
)
removeFn := func(enode.ID) {} // Noop
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -295,7 +304,8 @@ func TestPaidClientKickedOut(t *testing.T) {
removeFn := func(id enode.ID) {
kickedCh <- int(id[0])
}
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
pool.bt.SetExpirationTCs(0, 0)
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
@ -325,7 +335,8 @@ func TestConnectFreeClient(t *testing.T) {
clock mclock.Simulated
db = rawdb.NewMemoryDatabase()
)
pool := newClientPool(db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10))
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -341,7 +352,8 @@ func TestConnectFreeClientToFullPool(t *testing.T) {
db = rawdb.NewMemoryDatabase()
)
removeFn := func(enode.ID) {} // Noop
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -370,7 +382,8 @@ func TestFreeClientKickedOut(t *testing.T) {
kicked = make(chan int, 100)
)
removeFn := func(id enode.ID) { kicked <- int(id[0]) }
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -411,7 +424,8 @@ func TestPositiveBalanceCalculation(t *testing.T) {
kicked = make(chan int, 10)
)
removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -434,7 +448,8 @@ func TestDowngradePriorityClient(t *testing.T) {
kicked = make(chan int, 10)
)
removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop
pool := newClientPool(db, 1, defaultConnectedBias, &clock, removeFn)
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, removeFn)
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1, CapacityFactor: 0, RequestFactor: 1})
@ -468,7 +483,8 @@ func TestNegativeBalanceCalculation(t *testing.T) {
clock mclock.Simulated
db = rawdb.NewMemoryDatabase()
)
pool := newClientPool(db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool.ns.Start()
defer pool.stop()
pool.setLimits(10, uint64(10)) // Total capacity limit is 10
pool.setDefaultFactors(lps.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 1e-3, CapacityFactor: 0, RequestFactor: 1})
@ -503,7 +519,8 @@ func TestInactiveClient(t *testing.T) {
clock mclock.Simulated
db = rawdb.NewMemoryDatabase()
)
pool := newClientPool(db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool := newClientPool(testStateMachine(), db, 1, defaultConnectedBias, &clock, func(id enode.ID) {})
pool.ns.Start()
defer pool.stop()
pool.setLimits(2, uint64(2))

2
les/enr_entry.go
View file

@ -36,7 +36,7 @@ func (e lesEntry) ENRKey() string {
// setupDiscovery creates the node discovery source for the eth protocol.
func (eth *LightEthereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error) {
if /*cfg.NoDiscovery || */ len(eth.config.DiscoveryURLs) == 0 {
if cfg.NoDiscovery || len(eth.config.DiscoveryURLs) == 0 {
return nil, nil
}
client := dnsdisc.NewClient(dnsdisc.Config{})

6
les/lespay/server/prioritypool.go
View file

@ -253,12 +253,12 @@ func (pp *PriorityPool) SetActiveBias(bias time.Duration) {
pp.tryActivate()
}
// ActiveCapacity returns the total capacity of currently active nodes
func (pp *PriorityPool) ActiveCapacity() uint64 {
// Active returns the number and total capacity of currently active nodes
func (pp *PriorityPool) Active() (uint64, uint64) {
pp.lock.Lock()
defer pp.lock.Unlock()
return pp.activeCap
return pp.activeCount, pp.activeCap
}
// inactiveSetIndex callback updates ppNodeInfo item index in inactiveQueue

184
les/peer.go
View file

@ -126,7 +126,7 @@ type peerCommons struct {
frozen uint32 // Flag whether the peer is frozen.
announceType uint64 // New block announcement type.
serving uint32 // The status indicates the peer is served.
headInfo blockInfo // Latest block information.
headInfo blockInfo // Last announced block information.
// Background task queue for caching peer tasks and executing in order.
sendQueue *utils.ExecQueue
@ -255,6 +255,8 @@ func (p *peerCommons) handshake(td *big.Int, head common.Hash, headNum uint64, g
// Add some basic handshake fields
send = send.add("protocolVersion", uint64(p.version))
send = send.add("networkId", p.network)
// Note: the head info announced at handshake is only used in case of server peers
// but dummy values are still announced by clients for compatibility with older servers
send = send.add("headTd", td)
send = send.add("headHash", head)
send = send.add("headNum", headNum)
@ -273,24 +275,14 @@ func (p *peerCommons) handshake(td *big.Int, head common.Hash, headNum uint64, g
if size > allowedUpdateBytes {
return errResp(ErrRequestRejected, "")
}
var rGenesis, rHash common.Hash
var rVersion, rNetwork, rNum uint64
var rTd *big.Int
var rGenesis common.Hash
var rVersion, rNetwork uint64
if err := recv.get("protocolVersion", &rVersion); err != nil {
return err
}
if err := recv.get("networkId", &rNetwork); err != nil {
return err
}
if err := recv.get("headTd", &rTd); err != nil {
return err
}
if err := recv.get("headHash", &rHash); err != nil {
return err
}
if err := recv.get("headNum", &rNum); err != nil {
return err
}
if err := recv.get("genesisHash", &rGenesis); err != nil {
return err
}
@ -303,7 +295,6 @@ func (p *peerCommons) handshake(td *big.Int, head common.Hash, headNum uint64, g
if int(rVersion) != p.version {
return errResp(ErrProtocolVersionMismatch, "%d (!= %d)", rVersion, p.version)
}
p.headInfo = blockInfo{Hash: rHash, Number: rNum, Td: rTd}
if recvCallback != nil {
return recvCallback(recv)
}
@ -569,9 +560,11 @@ func (p *serverPeer) updateHead(hash common.Hash, number uint64, td *big.Int) {
}
// Handshake executes the les protocol handshake, negotiating version number,
// network IDs, difficulties, head and genesis blocks.
func (p *serverPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, server *LesServer) error {
return p.handshake(td, head, headNum, genesis, func(lists *keyValueList) {
// network IDs and genesis blocks.
func (p *serverPeer) Handshake(genesis common.Hash) error {
// Note: there is no need to share local head with a server but older servers still
// require these fields so we announce zero values.
return p.handshake(common.Big0, common.Hash{}, 0, genesis, func(lists *keyValueList) {
// Add some client-specific handshake fields
//
// Enable signed announcement randomly even the server is not trusted.
@ -581,6 +574,21 @@ func (p *serverPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge
}
*lists = (*lists).add("announceType", p.announceType)
}, func(recv keyValueMap) error {
var (
rHash common.Hash
rNum uint64
rTd *big.Int
)
if err := recv.get("headTd", &rTd); err != nil {
return err
}
if err := recv.get("headHash", &rHash); err != nil {
return err
}
if err := recv.get("headNum", &rNum); err != nil {
return err
}
p.headInfo = blockInfo{Hash: rHash, Number: rNum, Td: rTd}
if recv.get("serveChainSince", &p.chainSince) != nil {
p.onlyAnnounce = true
}
@ -937,6 +945,9 @@ func (p *clientPeer) freezeClient() {
// Handshake executes the les protocol handshake, negotiating version number,
// network IDs, difficulties, head and genesis blocks.
func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, server *LesServer) error {
// Note: clientPeer.headInfo should contain the last head announced to the client by us.
// The values announced in the handshake are dummy values for compatibility reasons and should be ignored.
p.headInfo = blockInfo{Hash: head, Number: headNum, Td: td}
return p.handshake(td, head, headNum, genesis, func(lists *keyValueList) {
// Add some information which services server can offer.
if !server.config.UltraLightOnlyAnnounce {
@ -1009,145 +1020,6 @@ type serverPeerSubscriber interface {
unregisterPeer(*serverPeer)
}
// clientPeerSubscriber is an interface to notify services about added or
// removed client peers
type clientPeerSubscriber interface {
registerPeer(*clientPeer)
unregisterPeer(*clientPeer)
}
// clientPeerSet represents the set of active client peers currently
// participating in the Light Ethereum sub-protocol.
type clientPeerSet struct {
peers map[string]*clientPeer
// subscribers is a batch of subscribers and peerset will notify
// these subscribers when the peerset changes(new client peer is
// added or removed)
subscribers []clientPeerSubscriber
closed bool
lock sync.RWMutex
}
// newClientPeerSet creates a new peer set to track the client peers.
func newClientPeerSet() *clientPeerSet {
return &clientPeerSet{peers: make(map[string]*clientPeer)}
}
// subscribe adds a service to be notified about added or removed
// peers and also register all active peers into the given service.
func (ps *clientPeerSet) subscribe(sub clientPeerSubscriber) {
ps.lock.Lock()
defer ps.lock.Unlock()
ps.subscribers = append(ps.subscribers, sub)
for _, p := range ps.peers {
sub.registerPeer(p)
}
}
// unSubscribe removes the specified service from the subscriber pool.
func (ps *clientPeerSet) unSubscribe(sub clientPeerSubscriber) {
ps.lock.Lock()
defer ps.lock.Unlock()
for i, s := range ps.subscribers {
if s == sub {
ps.subscribers = append(ps.subscribers[:i], ps.subscribers[i+1:]...)
return
}
}
}
// register adds a new peer into the peer set, or returns an error if the
// peer is already known.
func (ps *clientPeerSet) register(peer *clientPeer) error {
ps.lock.Lock()
defer ps.lock.Unlock()
if ps.closed {
return errClosed
}
if _, exist := ps.peers[peer.id]; exist {
return errAlreadyRegistered
}
ps.peers[peer.id] = peer
for _, sub := range ps.subscribers {
sub.registerPeer(peer)
}
return nil
}
// unregister removes a remote peer from the peer set, disabling any further
// actions to/from that particular entity. It also initiates disconnection
// at the networking layer.
func (ps *clientPeerSet) unregister(id string) error {
ps.lock.Lock()
defer ps.lock.Unlock()
p, ok := ps.peers[id]
if !ok {
return errNotRegistered
}
delete(ps.peers, id)
for _, sub := range ps.subscribers {
sub.unregisterPeer(p)
}
p.Peer.Disconnect(p2p.DiscRequested)
return nil
}
// ids returns a list of all registered peer IDs
func (ps *clientPeerSet) ids() []string {
ps.lock.RLock()
defer ps.lock.RUnlock()
var ids []string
for id := range ps.peers {
ids = append(ids, id)
}
return ids
}
// peer retrieves the registered peer with the given id.
func (ps *clientPeerSet) peer(id string) *clientPeer {
ps.lock.RLock()
defer ps.lock.RUnlock()
return ps.peers[id]
}
// len returns if the current number of peers in the set.
func (ps *clientPeerSet) len() int {
ps.lock.RLock()
defer ps.lock.RUnlock()
return len(ps.peers)
}
// allClientPeers returns all client peers in a list.
func (ps *clientPeerSet) allPeers() []*clientPeer {
ps.lock.RLock()
defer ps.lock.RUnlock()
list := make([]*clientPeer, 0, len(ps.peers))
for _, p := range ps.peers {
list = append(list, p)
}
return list
}
// close disconnects all peers. No new peers can be registered
// after close has returned.
func (ps *clientPeerSet) close() {
ps.lock.Lock()
defer ps.lock.Unlock()
for _, p := range ps.peers {
p.Disconnect(p2p.DiscQuitting)
}
ps.closed = true
}
// serverPeerSet represents the set of active server peers currently
// participating in the Light Ethereum sub-protocol.
type serverPeerSet struct {

10
les/protocol.go
View file

@ -174,12 +174,6 @@ var errorToString = map[int]string{
ErrMissingKey: "Key missing from list",
}
type announceBlock struct {
Hash common.Hash // Hash of one particular block being announced
Number uint64 // Number of one particular block being announced
Td *big.Int // Total difficulty of one particular block being announced
}
// announceData is the network packet for the block announcements.
type announceData struct {
Hash common.Hash // Hash of one particular block being announced
@ -199,7 +193,7 @@ func (a *announceData) sanityCheck() error {
// sign adds a signature to the block announcement by the given privKey
func (a *announceData) sign(privKey *ecdsa.PrivateKey) {
rlp, _ := rlp.EncodeToBytes(announceBlock{a.Hash, a.Number, a.Td})
rlp, _ := rlp.EncodeToBytes(blockInfo{a.Hash, a.Number, a.Td})
sig, _ := crypto.Sign(crypto.Keccak256(rlp), privKey)
a.Update = a.Update.add("sign", sig)
}
@ -210,7 +204,7 @@ func (a *announceData) checkSignature(id enode.ID, update keyValueMap) error {
if err := update.get("sign", &sig); err != nil {
return err
}
rlp, _ := rlp.EncodeToBytes(announceBlock{a.Hash, a.Number, a.Td})
rlp, _ := rlp.EncodeToBytes(blockInfo{a.Hash, a.Number, a.Td})
recPubkey, err := crypto.SigToPub(crypto.Keccak256(rlp), sig)
if err != nil {
return err

64
les/server.go
View file

@ -18,6 +18,7 @@ package les
import (
"crypto/ecdsa"
"reflect"
"time"
"github.com/ethereum/go-ethereum/common/mclock"
@ -31,17 +32,32 @@ import (
"github.com/ethereum/go-ethereum/p2p/discv5"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/enr"
"github.com/ethereum/go-ethereum/p2p/nodestate"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
var (
serverSetup = &nodestate.Setup{}
clientPeerField = serverSetup.NewField("clientPeer", reflect.TypeOf(&clientPeer{}))
clientInfoField = serverSetup.NewField("clientInfo", reflect.TypeOf(&clientInfo{}))
connAddressField = serverSetup.NewField("connAddr", reflect.TypeOf(""))
balanceTrackerSetup = lps.NewBalanceTrackerSetup(serverSetup)
priorityPoolSetup = lps.NewPriorityPoolSetup(serverSetup)
)
func init() {
balanceTrackerSetup.Connect(connAddressField, priorityPoolSetup.CapacityField)
priorityPoolSetup.Connect(balanceTrackerSetup.BalanceField, balanceTrackerSetup.UpdateFlag) // NodeBalance implements nodePriority
}
type LesServer struct {
lesCommons
ns *nodestate.NodeStateMachine
archiveMode bool // Flag whether the ethereum node runs in archive mode.
peers *clientPeerSet
serverset *serverSet
handler *serverHandler
broadcaster *broadcaster
lesTopics []discv5.Topic
privateKey *ecdsa.PrivateKey
@ -60,6 +76,7 @@ type LesServer struct {
}
func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesServer, error) {
ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup)
// Collect les protocol version information supported by local node.
lesTopics := make([]discv5.Topic, len(AdvertiseProtocolVersions))
for i, pv := range AdvertiseProtocolVersions {
@ -83,9 +100,9 @@ func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesSer
bloomTrieIndexer: light.NewBloomTrieIndexer(e.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency, true),
closeCh: make(chan struct{}),
},
ns: ns,
archiveMode: e.ArchiveMode(),
peers: newClientPeerSet(),
serverset: newServerSet(),
broadcaster: newBroadcaster(ns),
lesTopics: lesTopics,
fcManager: flowcontrol.NewClientManager(nil, &mclock.System{}),
servingQueue: newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100),
@ -116,7 +133,7 @@ func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesSer
srv.maxCapacity = totalRecharge
}
srv.fcManager.SetCapacityLimits(srv.minCapacity, srv.maxCapacity, srv.minCapacity*2)
srv.clientPool = newClientPool(srv.chainDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, func(id enode.ID) { go srv.peers.unregister(id.String()) })
srv.clientPool = newClientPool(ns, srv.chainDb, srv.minCapacity, defaultConnectedBias, mclock.System{}, srv.dropClient)
srv.clientPool.setDefaultFactors(lps.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1}, lps.PriceFactors{TimeFactor: 0, CapacityFactor: 1, RequestFactor: 1})
checkpoint := srv.latestLocalCheckpoint()
@ -130,6 +147,13 @@ func NewLesServer(node *node.Node, e *eth.Ethereum, config *eth.Config) (*LesSer
node.RegisterAPIs(srv.APIs())
node.RegisterLifecycle(srv)
// disconnect all peers at nsm shutdown
ns.SubscribeField(clientPeerField, func(node *enode.Node, state nodestate.Flags, oldValue, newValue interface{}) {
if state.Equals(serverSetup.OfflineFlag()) && oldValue != nil {
oldValue.(*clientPeer).Peer.Disconnect(p2p.DiscRequested)
}
})
ns.Start()
return srv, nil
}
@ -158,7 +182,7 @@ func (s *LesServer) APIs() []rpc.API {
func (s *LesServer) Protocols() []p2p.Protocol {
ps := s.makeProtocols(ServerProtocolVersions, s.handler.runPeer, func(id enode.ID) interface{} {
if p := s.peers.peer(id.String()); p != nil {
if p := s.getClient(id); p != nil {
return p.Info()
}
return nil
@ -173,6 +197,7 @@ func (s *LesServer) Protocols() []p2p.Protocol {
// Start starts the LES server
func (s *LesServer) Start() error {
s.privateKey = s.p2pSrv.PrivateKey
s.broadcaster.setSignerKey(s.privateKey)
s.handler.start()
s.wg.Add(1)
@ -198,19 +223,11 @@ func (s *LesServer) Start() error {
func (s *LesServer) Stop() error {
close(s.closeCh)
// Disconnect existing connections with other LES servers.
s.serverset.close()
// Disconnect existing sessions.
// This also closes the gate for any new registrations on the peer set.
// sessions which are already established but not added to pm.peers yet
// will exit when they try to register.
s.peers.close()
s.clientPool.stop()
s.ns.Stop()
s.fcManager.Stop()
s.costTracker.stop()
s.handler.stop()
s.clientPool.stop() // client pool should be closed after handler.
s.servingQueue.stop()
// Note, bloom trie indexer is closed by parent bloombits indexer.
@ -279,3 +296,18 @@ func (s *LesServer) capacityManagement() {
}
}
}
func (s *LesServer) getClient(id enode.ID) *clientPeer {
if node := s.ns.GetNode(id); node != nil {
if p, ok := s.ns.GetField(node, clientPeerField).(*clientPeer); ok {
return p
}
}
return nil
}
func (s *LesServer) dropClient(id enode.ID) {
if p := s.getClient(id); p != nil {
p.Peer.Disconnect(p2p.DiscRequested)
}
}

152
les/server_handler.go
View file

@ -17,6 +17,7 @@
package les
import (
"crypto/ecdsa"
"encoding/binary"
"encoding/json"
"errors"
@ -36,6 +37,8 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/nodestate"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
@ -91,7 +94,7 @@ func newServerHandler(server *LesServer, blockchain *core.BlockChain, chainDb et
// start starts the server handler.
func (h *serverHandler) start() {
h.wg.Add(1)
go h.broadcastHeaders()
go h.broadcastLoop()
}
// stop stops the server handler.
@ -123,47 +126,61 @@ func (h *serverHandler) handle(p *clientPeer) error {
p.Log().Debug("Light Ethereum handshake failed", "err", err)
return err
}
if p.server {
if err := h.server.serverset.register(p); err != nil {
return err
// Reject the duplicated peer, otherwise register it to peerset.
var registered bool
if err := h.server.ns.Operation(func() {
if h.server.ns.GetField(p.Node(), clientPeerField) != nil {
registered = true
} else {
h.server.ns.SetFieldSub(p.Node(), clientPeerField, p)
}
}); err != nil {
return err
}
if registered {
return errAlreadyRegistered
}
defer func() {
h.server.ns.SetField(p.Node(), clientPeerField, nil)
if p.fcClient != nil { // is nil when connecting another server
p.fcClient.Disconnect()
}
}()
if p.server {
// connected to another server, no messages expected, just wait for disconnection
_, err := p.rw.ReadMsg()
return err
}
// Reject light clients if server is not synced.
//
// Put this checking here, so that "non-synced" les-server peers are still allowed
// to keep the connection.
if !h.synced() {
p.Log().Debug("Light server not synced, rejecting peer")
return p2p.DiscRequested
}
defer p.fcClient.Disconnect()
// Disconnect the inbound peer if it's rejected by clientPool
if cap, err := h.server.clientPool.connect(p); cap != p.fcParams.MinRecharge || err != nil {
p.Log().Debug("Light Ethereum peer rejected", "err", errFullClientPool)
return errFullClientPool
}
p.balance, _ = h.server.clientPool.ns.GetField(p.Node(), h.server.clientPool.BalanceField).(*lps.NodeBalance)
p.balance, _ = h.server.ns.GetField(p.Node(), h.server.clientPool.BalanceField).(*lps.NodeBalance)
if p.balance == nil {
return p2p.DiscRequested
}
// Register the peer locally
if err := h.server.peers.register(p); err != nil {
h.server.clientPool.disconnect(p)
p.Log().Error("Light Ethereum peer registration failed", "err", err)
return err
}
clientConnectionGauge.Update(int64(h.server.peers.len()))
activeCount, _ := h.server.clientPool.pp.Active()
clientConnectionGauge.Update(int64(activeCount))
var wg sync.WaitGroup // Wait group used to track all in-flight task routines.
connectedAt := mclock.Now()
defer func() {
wg.Wait() // Ensure all background task routines have exited.
h.server.peers.unregister(p.id)
h.server.clientPool.disconnect(p)
p.balance = nil
clientConnectionGauge.Update(int64(h.server.peers.len()))
activeCount, _ := h.server.clientPool.pp.Active()
clientConnectionGauge.Update(int64(activeCount))
connectionTimer.Update(time.Duration(mclock.Now() - connectedAt))
}()
// Mark the peer starts to be served.
@ -334,7 +351,6 @@ func (h *serverHandler) handleMsg(p *clientPeer, wg *sync.WaitGroup) error {
origin = h.blockchain.GetHeaderByNumber(query.Origin.Number)
}
if origin == nil {
p.bumpInvalid()
break
}
headers = append(headers, origin)
@ -911,11 +927,11 @@ func (h *serverHandler) txStatus(hash common.Hash) light.TxStatus {
return stat
}
// broadcastHeaders broadcasts new block information to all connected light
// broadcastLoop broadcasts new block information to all connected light
// clients. According to the agreement between client and server, server should
// only broadcast new announcement if the total difficulty is higher than the
// last one. Besides server will add the signature if client requires.
func (h *serverHandler) broadcastHeaders() {
func (h *serverHandler) broadcastLoop() {
defer h.wg.Done()
headCh := make(chan core.ChainHeadEvent, 10)
@ -929,10 +945,6 @@ func (h *serverHandler) broadcastHeaders() {
for {
select {
case ev := <-headCh:
peers := h.server.peers.allPeers()
if len(peers) == 0 {
continue
}
header := ev.Block.Header()
hash, number := header.Hash(), header.Number.Uint64()
td := h.blockchain.GetTd(hash, number)
@ -944,33 +956,79 @@ func (h *serverHandler) broadcastHeaders() {
reorg = lastHead.Number.Uint64() - rawdb.FindCommonAncestor(h.chainDb, header, lastHead).Number.Uint64()
}
lastHead, lastTd = header, td
log.Debug("Announcing block to peers", "number", number, "hash", hash, "td", td, "reorg", reorg)
var (
signed bool
signedAnnounce announceData
)
announce := announceData{Hash: hash, Number: number, Td: td, ReorgDepth: reorg}
for _, p := range peers {
p := p
switch p.announceType {
case announceTypeSimple:
if !p.queueSend(func() { p.sendAnnounce(announce) }) {
log.Debug("Drop announcement because queue is full", "number", number, "hash", hash)
}
case announceTypeSigned:
if !signed {
signedAnnounce = announce
signedAnnounce.sign(h.server.privateKey)
signed = true
}
if !p.queueSend(func() { p.sendAnnounce(signedAnnounce) }) {
log.Debug("Drop announcement because queue is full", "number", number, "hash", hash)
}
}
}
h.server.broadcaster.broadcast(announceData{Hash: hash, Number: number, Td: td, ReorgDepth: reorg})
case <-h.closeCh:
return
}
}
}
// broadcaster sends new header announcements to active client peers
type broadcaster struct {
ns *nodestate.NodeStateMachine
privateKey *ecdsa.PrivateKey
lastAnnounce, signedAnnounce announceData
}
// newBroadcaster creates a new broadcaster
func newBroadcaster(ns *nodestate.NodeStateMachine) *broadcaster {
b := &broadcaster{ns: ns}
ns.SubscribeState(priorityPoolSetup.ActiveFlag, func(node *enode.Node, oldState, newState nodestate.Flags) {
if newState.Equals(priorityPoolSetup.ActiveFlag) {
// send last announcement to activated peers
b.sendTo(node)
}
})
return b
}
// setSignerKey sets the signer key for signed announcements. Should be called before
// starting the protocol handler.
func (b *broadcaster) setSignerKey(privateKey *ecdsa.PrivateKey) {
b.privateKey = privateKey
}
// broadcast sends the given announcements to all active peers
func (b *broadcaster) broadcast(announce announceData) {
b.ns.Operation(func() {
// iterate in an Operation to ensure that the active set does not change while iterating
b.lastAnnounce = announce
b.ns.ForEach(priorityPoolSetup.ActiveFlag, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) {
b.sendTo(node)
})
})
}
// sendTo sends the most recent announcement to the given node unless the same or higher Td
// announcement has already been sent.
func (b *broadcaster) sendTo(node *enode.Node) {
if b.lastAnnounce.Td == nil {
return
}
if p, _ := b.ns.GetField(node, clientPeerField).(*clientPeer); p != nil {
if p.headInfo.Td == nil || b.lastAnnounce.Td.Cmp(p.headInfo.Td) > 0 {
announce := b.lastAnnounce
switch p.announceType {
case announceTypeSimple:
if !p.queueSend(func() { p.sendAnnounce(announce) }) {
log.Debug("Drop announcement because queue is full", "number", announce.Number, "hash", announce.Hash)
} else {
log.Debug("Sent announcement", "number", announce.Number, "hash", announce.Hash)
}
case announceTypeSigned:
if b.signedAnnounce.Hash != b.lastAnnounce.Hash {
b.signedAnnounce = b.lastAnnounce
b.signedAnnounce.sign(b.privateKey)
}
announce := b.signedAnnounce
if !p.queueSend(func() { p.sendAnnounce(announce) }) {
log.Debug("Drop announcement because queue is full", "number", announce.Number, "hash", announce.Hash)
} else {
log.Debug("Sent announcement", "number", announce.Number, "hash", announce.Hash)
}
}
p.headInfo = blockInfo{b.lastAnnounce.Hash, b.lastAnnounce.Number, b.lastAnnounce.Td}
}
}
}

16
les/test_helper.go
View file

@ -46,6 +46,7 @@ import (
"github.com/ethereum/go-ethereum/light"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/nodestate"
"github.com/ethereum/go-ethereum/params"
)
@ -227,7 +228,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index
return client.handler
}
func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Database, peers *clientPeerSet, clock mclock.Clock) (*serverHandler, *backends.SimulatedBackend) {
func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Database, clock mclock.Clock) (*serverHandler, *backends.SimulatedBackend) {
var (
gspec = core.Genesis{
Config: params.AllEthashProtocolChanges,
@ -263,6 +264,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
}
oracle = checkpointoracle.New(checkpointConfig, getLocal)
}
ns := nodestate.NewNodeStateMachine(nil, nil, mclock.System{}, serverSetup)
server := &LesServer{
lesCommons: lesCommons{
genesis: genesis.Hash(),
@ -274,7 +276,8 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
oracle: oracle,
closeCh: make(chan struct{}),
},
peers: peers,
ns: ns,
broadcaster: newBroadcaster(ns),
servingQueue: newServingQueue(int64(time.Millisecond*10), 1),
defParams: flowcontrol.ServerParams{
BufLimit: testBufLimit,
@ -284,13 +287,14 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
}
server.costTracker, server.minCapacity = newCostTracker(db, server.config)
server.costTracker.testCostList = testCostList(0) // Disable flow control mechanism.
server.clientPool = newClientPool(db, testBufRecharge, defaultConnectedBias, clock, func(id enode.ID) {})
server.clientPool = newClientPool(ns, db, testBufRecharge, defaultConnectedBias, clock, func(id enode.ID) {})
server.clientPool.setLimits(10000, 10000) // Assign enough capacity for clientpool
server.handler = newServerHandler(server, simulation.Blockchain(), db, txpool, func() bool { return true })
if server.oracle != nil {
server.oracle.Start(simulation)
}
server.servingQueue.setThreads(4)
ns.Start()
server.handler.start()
return server.handler, simulation
}
@ -463,7 +467,7 @@ func newServerEnv(t *testing.T, blocks int, protocol int, callback indexerCallba
if simClock {
clock = &mclock.Simulated{}
}
handler, b := newTestServerHandler(blocks, indexers, db, newClientPeerSet(), clock)
handler, b := newTestServerHandler(blocks, indexers, db, clock)
var peer *testPeer
if newPeer {
@ -502,7 +506,7 @@ func newServerEnv(t *testing.T, blocks int, protocol int, callback indexerCallba
func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexerCallback, ulcServers []string, ulcFraction int, simClock bool, connect bool, disablePruning bool) (*testServer, *testClient, func()) {
sdb, cdb := rawdb.NewMemoryDatabase(), rawdb.NewMemoryDatabase()
speers, cpeers := newServerPeerSet(), newClientPeerSet()
speers := newServerPeerSet()
var clock mclock.Clock = &mclock.System{}
if simClock {
@ -519,7 +523,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, callback indexer
ccIndexer, cbIndexer, cbtIndexer := cIndexers[0], cIndexers[1], cIndexers[2]
odr.SetIndexers(ccIndexer, cbIndexer, cbtIndexer)
server, b := newTestServerHandler(blocks, sindexers, sdb, cpeers, clock)
server, b := newTestServerHandler(blocks, sindexers, sdb, clock)
client := newTestClientHandler(b, odr, cIndexers, cdb, speers, ulcServers, ulcFraction)
scIndexer.Start(server.blockchain)

23
miner/miner_test.go
View file

@ -22,7 +22,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
@ -31,7 +31,6 @@ import (
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
@ -243,26 +242,20 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) {
if err != nil {
t.Fatalf("can't create new chain config: %v", err)
}
// Create event Mux
mux := new(event.TypeMux)
// Create consensus engine
engine := ethash.New(ethash.Config{}, []string{}, false)
engine.SetThreads(-1)
// Create isLocalBlock
isLocalBlock := func(block *types.Block) bool {
return true
}
engine := clique.New(chainConfig.Clique, chainDB)
// Create Ethereum backend
limit := uint64(1000)
bc, err := core.NewBlockChain(chainDB, new(core.CacheConfig), chainConfig, engine, vm.Config{}, isLocalBlock, &limit)
bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("can't create new chain %v", err)
}
statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil)
statedb, _ := state.New(common.Hash{}, state.NewDatabase(chainDB), nil)
blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)}
pool := core.NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
pool := core.NewTxPool(testTxPoolConfig, chainConfig, blockchain)
backend := NewMockBackend(bc, pool)
// Create event Mux
mux := new(event.TypeMux)
// Create Miner
return New(backend, &config, chainConfig, mux, engine, isLocalBlock), mux
return New(backend, &config, chainConfig, mux, engine, nil), mux
}

56
oss-fuzz.sh Normal file
View file

@ -0,0 +1,56 @@
#/bin/bash -eu
# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
# This file is for integration with Google OSS-Fuzz.
# The following ENV variables are available when executing on OSS-fuzz:
#
# /out/ $OUT Directory to store build artifacts (fuzz targets, dictionaries, options files, seed corpus archives).
# /src/ $SRC Directory to checkout source files.
# /work/ $WORK Directory to store intermediate files.
#
# $CC, $CXX, $CCC The C and C++ compiler binaries.
# $CFLAGS, $CXXFLAGS C and C++ compiler flags.
# $LIB_FUZZING_ENGINE C++ compiler argument to link fuzz target against the prebuilt engine library (e.g. libFuzzer).
function compile_fuzzer {
path=$SRC/go-ethereum/$1
func=$2
fuzzer=$3
echo "Building $fuzzer"
(cd $path && \
go-fuzz -func $func -o $WORK/$fuzzer.a . && \
echo "First stage built OK" && \
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $WORK/$fuzzer.a -o $OUT/$fuzzer && \
echo "Second stage built ok" )
}
compile_fuzzer common/bitutil Fuzz fuzzBitutilCompress
compile_fuzzer crypto/bn256 FuzzAdd fuzzBn256Add
compile_fuzzer crypto/bn256 FuzzMul fuzzBn256Mul
compile_fuzzer crypto/bn256 FuzzPair fuzzBn256Pair
compile_fuzzer core/vm/runtime Fuzz fuzzVmRuntime
compile_fuzzer crypto/blake2b Fuzz fuzzBlake2b
compile_fuzzer tests/fuzzers/keystore Fuzz fuzzKeystore
compile_fuzzer tests/fuzzers/txfetcher Fuzz fuzzTxfetcher
compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp
compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie
compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie
# This doesn't work very well @TODO
#compile_fuzzertests/fuzzers/abi Fuzz fuzzAbi

54
p2p/simulations/adapters/exec.go
View file

@ -184,7 +184,19 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) {
if err != nil {
return fmt.Errorf("error generating node config: %s", err)
}
// expose the admin namespace via websocket if it's not enabled
exposed := confCopy.Stack.WSExposeAll
if !exposed {
for _, api := range confCopy.Stack.WSModules {
if api == "admin" {
exposed = true
break
}
}
}
if !exposed {
confCopy.Stack.WSModules = append(confCopy.Stack.WSModules, "admin")
}
// start the one-shot server that waits for startup information
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
@ -362,13 +374,44 @@ type execNodeConfig struct {
PeerAddrs map[string]string `json:"peer_addrs,omitempty"`
}
func initLogging() {
// Initialize the logging by default first.
glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()))
glogger.Verbosity(log.LvlInfo)
log.Root().SetHandler(glogger)
confEnv := os.Getenv(envNodeConfig)
if confEnv == "" {
return
}
var conf execNodeConfig
if err := json.Unmarshal([]byte(confEnv), &conf); err != nil {
return
}
var writer = os.Stderr
if conf.Node.LogFile != "" {
logWriter, err := os.Create(conf.Node.LogFile)
if err != nil {
return
}
writer = logWriter
}
var verbosity = log.LvlInfo
if conf.Node.LogVerbosity <= log.LvlTrace && conf.Node.LogVerbosity >= log.LvlCrit {
verbosity = conf.Node.LogVerbosity
}
// Reinitialize the logger
glogger = log.NewGlogHandler(log.StreamHandler(writer, log.TerminalFormat(true)))
glogger.Verbosity(verbosity)
log.Root().SetHandler(glogger)
}
// execP2PNode starts a simulation node when the current binary is executed with
// argv[0] being "p2p-node", reading the service / ID from argv[1] / argv[2]
// and the node config from an environment variable.
func execP2PNode() {
glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat()))
glogger.Verbosity(log.LvlInfo)
log.Root().SetHandler(glogger)
initLogging()
statusURL := os.Getenv(envStatusURL)
if statusURL == "" {
log.Crit("missing " + envStatusURL)
@ -380,7 +423,7 @@ func execP2PNode() {
if stackErr != nil {
status.Err = stackErr.Error()
} else {
status.WSEndpoint = "ws://" + stack.WSEndpoint()
status.WSEndpoint = stack.WSEndpoint()
status.NodeInfo = stack.Server().NodeInfo()
}
@ -454,7 +497,6 @@ func startExecNodeStack() (*node.Node, error) {
return nil, err
}
services[name] = service
stack.RegisterLifecycle(service)
}
// Add the snapshot API.

6
p2p/simulations/adapters/inproc.go
View file

@ -99,8 +99,9 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) {
Dialer: s,
EnableMsgEvents: config.EnableMsgEvents,
},
NoUSB: true,
Logger: log.New("node.id", id.String()),
ExternalSigner: config.ExternalSigner,
NoUSB: true,
Logger: log.New("node.id", id.String()),
})
if err != nil {
return nil, err
@ -263,7 +264,6 @@ func (sn *SimNode) Start(snapshots map[string][]byte) error {
continue
}
sn.running[name] = service
sn.node.RegisterLifecycle(service)
}
})
if regErr != nil {

27
p2p/simulations/adapters/types.go
View file

@ -107,6 +107,9 @@ type NodeConfig struct {
// These values need to be checked and acted upon by node Services
Properties []string
// ExternalSigner specifies an external URI for a clef-type signer
ExternalSigner string
// Enode
node *enode.Node
@ -117,6 +120,17 @@ type NodeConfig struct {
Reachable func(id enode.ID) bool
Port uint16
// LogFile is the log file name of the p2p node at runtime.
//
// The default value is empty so that the default log writer
// is the system standard output.
LogFile string
// LogVerbosity is the log verbosity of the p2p node at runtime.
//
// The default verbosity is INFO.
LogVerbosity log.Lvl
}
// nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding
@ -125,10 +139,12 @@ type nodeConfigJSON struct {
ID string `json:"id"`
PrivateKey string `json:"private_key"`
Name string `json:"name"`
Services []string `json:"services"`
Lifecycles []string `json:"lifecycles"`
Properties []string `json:"properties"`
EnableMsgEvents bool `json:"enable_msg_events"`
Port uint16 `json:"port"`
LogFile string `json:"logfile"`
LogVerbosity int `json:"log_verbosity"`
}
// MarshalJSON implements the json.Marshaler interface by encoding the config
@ -137,10 +153,12 @@ func (n *NodeConfig) MarshalJSON() ([]byte, error) {
confJSON := nodeConfigJSON{
ID: n.ID.String(),
Name: n.Name,
Services: n.Lifecycles,
Lifecycles: n.Lifecycles,
Properties: n.Properties,
Port: n.Port,
EnableMsgEvents: n.EnableMsgEvents,
LogFile: n.LogFile,
LogVerbosity: int(n.LogVerbosity),
}
if n.PrivateKey != nil {
confJSON.PrivateKey = hex.EncodeToString(crypto.FromECDSA(n.PrivateKey))
@ -175,10 +193,12 @@ func (n *NodeConfig) UnmarshalJSON(data []byte) error {
}
n.Name = confJSON.Name
n.Lifecycles = confJSON.Services
n.Lifecycles = confJSON.Lifecycles
n.Properties = confJSON.Properties
n.Port = confJSON.Port
n.EnableMsgEvents = confJSON.EnableMsgEvents
n.LogFile = confJSON.LogFile
n.LogVerbosity = log.Lvl(confJSON.LogVerbosity)
return nil
}
@ -208,6 +228,7 @@ func RandomNodeConfig() *NodeConfig {
Name: fmt.Sprintf("node_%s", enodId.String()),
Port: port,
EnableMsgEvents: true,
LogVerbosity: log.LvlInfo,
}
}

8
params/bootnodes.go
View file

@ -67,10 +67,10 @@ var GoerliBootnodes = []string{
"enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303",
}
// YoloV1Bootnodes are the enode URLs of the P2P bootstrap nodes running on the
// YOLOv1 ephemeral test network.
var YoloV1Bootnodes = []string{
"enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@35.178.210.161:30303",
// YoloV2Bootnodes are the enode URLs of the P2P bootstrap nodes running on the
// YOLOv2 ephemeral test network.
var YoloV2Bootnodes = []string{
"enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303",
}
const dnsPrefix = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@"

33
params/config.go
View file

@ -31,7 +31,8 @@ var (
RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177")
GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a")
YoloV1GenesisHash = common.HexToHash("0xc3fd235071f24f93865b0850bd2a2119b30f7224d18a0e34c7bbf549ad7e3d36")
// TODO: update with yolov2 values
YoloV2GenesisHash = common.HexToHash("0x498a7239036dd2cd09e2bb8a80922b78632017958c332b42044c250d603a8a3e")
)
// TrustedCheckpoints associates each known checkpoint with the genesis hash of
@ -213,9 +214,9 @@ var (
Threshold: 2,
}
// YoloV1ChainConfig contains the chain parameters to run a node on the YOLOv1 test network.
YoloV1ChainConfig = &ChainConfig{
ChainID: big.NewInt(133519467574833),
// YoloV2ChainConfig contains the chain parameters to run a node on the YOLOv2 test network.
YoloV2ChainConfig = &ChainConfig{
ChainID: big.NewInt(133519467574834),
HomesteadBlock: big.NewInt(0),
DAOForkBlock: nil,
DAOForkSupport: true,
@ -227,7 +228,7 @@ var (
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: nil,
YoloV1Block: big.NewInt(0),
YoloV2Block: big.NewInt(0),
Clique: &CliqueConfig{
Period: 15,
Epoch: 30000,
@ -320,7 +321,7 @@ type ChainConfig struct {
IstanbulBlock *big.Int `json:"istanbulBlock,omitempty"` // Istanbul switch block (nil = no fork, 0 = already on istanbul)
MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated)
YoloV1Block *big.Int `json:"yoloV1Block,omitempty"` // YOLO v1: https://github.com/ethereum/EIPs/pull/2657 (Ephemeral testnet)
YoloV2Block *big.Int `json:"yoloV2Block,omitempty"` // YOLO v2: Gas repricings TODO @holiman add EIP references
EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated)
// Various consensus engines
@ -358,7 +359,7 @@ func (c *ChainConfig) String() string {
default:
engine = "unknown"
}
return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, YOLO v1: %v, Engine: %v}",
return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, YOLO v2: %v, Engine: %v}",
c.ChainID,
c.HomesteadBlock,
c.DAOForkBlock,
@ -371,7 +372,7 @@ func (c *ChainConfig) String() string {
c.PetersburgBlock,
c.IstanbulBlock,
c.MuirGlacierBlock,
c.YoloV1Block,
c.YoloV2Block,
engine,
)
}
@ -428,9 +429,9 @@ func (c *ChainConfig) IsIstanbul(num *big.Int) bool {
return isForked(c.IstanbulBlock, num)
}
// IsYoloV1 returns whether num is either equal to the YoloV1 fork block or greater.
func (c *ChainConfig) IsYoloV1(num *big.Int) bool {
return isForked(c.YoloV1Block, num)
// IsYoloV2 returns whether num is either equal to the YoloV1 fork block or greater.
func (c *ChainConfig) IsYoloV2(num *big.Int) bool {
return isForked(c.YoloV2Block, num)
}
// IsEWASM returns whether num represents a block number after the EWASM fork
@ -476,7 +477,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error {
{name: "petersburgBlock", block: c.PetersburgBlock},
{name: "istanbulBlock", block: c.IstanbulBlock},
{name: "muirGlacierBlock", block: c.MuirGlacierBlock, optional: true},
{name: "yoloV1Block", block: c.YoloV1Block},
{name: "yoloV2Block", block: c.YoloV2Block},
} {
if lastFork.name != "" {
// Next one must be higher number
@ -540,8 +541,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi
if isForkIncompatible(c.MuirGlacierBlock, newcfg.MuirGlacierBlock, head) {
return newCompatError("Muir Glacier fork block", c.MuirGlacierBlock, newcfg.MuirGlacierBlock)
}
if isForkIncompatible(c.YoloV1Block, newcfg.YoloV1Block, head) {
return newCompatError("YOLOv1 fork block", c.YoloV1Block, newcfg.YoloV1Block)
if isForkIncompatible(c.YoloV2Block, newcfg.YoloV2Block, head) {
return newCompatError("YOLOv2 fork block", c.YoloV2Block, newcfg.YoloV2Block)
}
if isForkIncompatible(c.EWASMBlock, newcfg.EWASMBlock, head) {
return newCompatError("ewasm fork block", c.EWASMBlock, newcfg.EWASMBlock)
@ -613,7 +614,7 @@ type Rules struct {
ChainID *big.Int
IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool
IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool
IsYoloV1 bool
IsYoloV2 bool
}
// Rules ensures c's ChainID is not nil.
@ -632,6 +633,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules {
IsConstantinople: c.IsConstantinople(num),
IsPetersburg: c.IsPetersburg(num),
IsIstanbul: c.IsIstanbul(num),
IsYoloV1: c.IsYoloV1(num),
IsYoloV2: c.IsYoloV2(num),
}
}

2
params/version.go
View file

@ -23,7 +23,7 @@ import (
const (
VersionMajor = 1 // Major version component of the current release
VersionMinor = 9 // Minor version component of the current release
VersionPatch = 23 // Patch version component of the current release
VersionPatch = 24 // Patch version component of the current release
VersionMeta = "stable" // Version metadata to append to the version string
)

71
rlp/raw.go
View file

@ -180,3 +180,74 @@ func readSize(b []byte, slen byte) (uint64, error) {
}
return s, nil
}
// AppendUint64 appends the RLP encoding of i to b, and returns the resulting slice.
func AppendUint64(b []byte, i uint64) []byte {
if i == 0 {
return append(b, 0x80)
} else if i < 128 {
return append(b, byte(i))
}
switch {
case i < (1 << 8):
return append(b, 0x81, byte(i))
case i < (1 << 16):
return append(b, 0x82,
byte(i>>8),
byte(i),
)
case i < (1 << 24):
return append(b, 0x83,
byte(i>>16),
byte(i>>8),
byte(i),
)
case i < (1 << 32):
return append(b, 0x84,
byte(i>>24),
byte(i>>16),
byte(i>>8),
byte(i),
)
case i < (1 << 40):
return append(b, 0x85,
byte(i>>32),
byte(i>>24),
byte(i>>16),
byte(i>>8),
byte(i),
)
case i < (1 << 48):
return append(b, 0x86,
byte(i>>40),
byte(i>>32),
byte(i>>24),
byte(i>>16),
byte(i>>8),
byte(i),
)
case i < (1 << 56):
return append(b, 0x87,
byte(i>>48),
byte(i>>40),
byte(i>>32),
byte(i>>24),
byte(i>>16),
byte(i>>8),
byte(i),
)
default:
return append(b, 0x88,
byte(i>>56),
byte(i>>48),
byte(i>>40),
byte(i>>32),
byte(i>>24),
byte(i>>16),
byte(i>>8),
byte(i),
)
}
}

38
rlp/raw_test.go
View file

@ -21,6 +21,7 @@ import (
"io"
"reflect"
"testing"
"testing/quick"
)
func TestCountValues(t *testing.T) {
@ -239,3 +240,40 @@ func TestReadSize(t *testing.T) {
}
}
}
func TestAppendUint64(t *testing.T) {
tests := []struct {
input uint64
slice []byte
output string
}{
{0, nil, "80"},
{1, nil, "01"},
{2, nil, "02"},
{127, nil, "7F"},
{128, nil, "8180"},
{129, nil, "8181"},
{0xFFFFFF, nil, "83FFFFFF"},
{127, []byte{1, 2, 3}, "0102037F"},
{0xFFFFFF, []byte{1, 2, 3}, "01020383FFFFFF"},
}
for _, test := range tests {
x := AppendUint64(test.slice, test.input)
if !bytes.Equal(x, unhex(test.output)) {
t.Errorf("AppendUint64(%v, %d): got %x, want %s", test.slice, test.input, x, test.output)
}
}
}
func TestAppendUint64Random(t *testing.T) {
fn := func(i uint64) bool {
enc, _ := EncodeToBytes(i)
encAppend := AppendUint64(nil, i)
return bytes.Equal(enc, encAppend)
}
config := quick.Config{MaxCountScale: 50}
if err := quick.Check(fn, &config); err != nil {
t.Fatal(err)
}
}

23
tests/fuzzers/stacktrie/debug/main.go Normal file
View file

@ -0,0 +1,23 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"github.com/ethereum/go-ethereum/tests/fuzzers/stacktrie"
)
func main() {
if len(os.Args) != 2 {
fmt.Fprintf(os.Stderr, "Usage: debug <file>")
os.Exit(1)
}
crasher := os.Args[1]
data, err := ioutil.ReadFile(crasher)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err)
os.Exit(1)
}
stacktrie.Debug(data)
}

197
tests/fuzzers/stacktrie/trie_fuzzer.go Normal file
View file

@ -0,0 +1,197 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package stacktrie
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash"
"io"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3"
)
type fuzzer struct {
input io.Reader
exhausted bool
debugging bool
}
func (f *fuzzer) read(size int) []byte {
out := make([]byte, size)
if _, err := f.input.Read(out); err != nil {
f.exhausted = true
}
return out
}
func (f *fuzzer) readSlice(min, max int) []byte {
var a uint16
binary.Read(f.input, binary.LittleEndian, &a)
size := min + int(a)%(max-min)
out := make([]byte, size)
if _, err := f.input.Read(out); err != nil {
f.exhausted = true
}
return out
}
// spongeDb is a dummy db backend which accumulates writes in a sponge
type spongeDb struct {
sponge hash.Hash
debug bool
}
func (s *spongeDb) Has(key []byte) (bool, error) { panic("implement me") }
func (s *spongeDb) Get(key []byte) ([]byte, error) { return nil, errors.New("no such elem") }
func (s *spongeDb) Delete(key []byte) error { panic("implement me") }
func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} }
func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") }
func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") }
func (s *spongeDb) Close() error { return nil }
func (s *spongeDb) Put(key []byte, value []byte) error {
if s.debug {
fmt.Printf("db.Put %x : %x\n", key, value)
}
s.sponge.Write(key)
s.sponge.Write(value)
return nil
}
func (s *spongeDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { panic("implement me") }
// spongeBatch is a dummy batch which immediately writes to the underlying spongedb
type spongeBatch struct {
db *spongeDb
}
func (b *spongeBatch) Put(key, value []byte) error {
b.db.Put(key, value)
return nil
}
func (b *spongeBatch) Delete(key []byte) error { panic("implement me") }
func (b *spongeBatch) ValueSize() int { return 100 }
func (b *spongeBatch) Write() error { return nil }
func (b *spongeBatch) Reset() {}
func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil }
type kv struct {
k, v []byte
}
type kvs []kv
func (k kvs) Len() int {
return len(k)
}
func (k kvs) Less(i, j int) bool {
return bytes.Compare(k[i].k, k[j].k) < 0
}
func (k kvs) Swap(i, j int) {
k[j], k[i] = k[i], k[j]
}
// The function must return
// 1 if the fuzzer should increase priority of the
// given input during subsequent fuzzing (for example, the input is lexically
// correct and was parsed successfully);
// -1 if the input must not be added to corpus even if gives new coverage; and
// 0 otherwise
// other values are reserved for future use.
func Fuzz(data []byte) int {
f := fuzzer{
input: bytes.NewReader(data),
exhausted: false,
}
return f.fuzz()
}
func Debug(data []byte) int {
f := fuzzer{
input: bytes.NewReader(data),
exhausted: false,
debugging: true,
}
return f.fuzz()
}
func (f *fuzzer) fuzz() int {
// This spongeDb is used to check the sequence of disk-db-writes
var (
spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbA = trie.NewDatabase(spongeA)
trieA, _ = trie.New(common.Hash{}, dbA)
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
trieB = trie.NewStackTrie(spongeB)
vals kvs
useful bool
maxElements = 10000
)
// Fill the trie with elements
for i := 0; !f.exhausted && i < maxElements; i++ {
k := f.read(32)
v := f.readSlice(1, 500)
if f.exhausted {
// If it was exhausted while reading, the value may be all zeroes,
// thus 'deletion' which is not supported on stacktrie
break
}
vals = append(vals, kv{k: k, v: v})
trieA.Update(k, v)
useful = true
}
if !useful {
return 0
}
// Flush trie -> database
rootA, err := trieA.Commit(nil)
if err != nil {
panic(err)
}
// Flush memdb -> disk (sponge)
dbA.Commit(rootA, false, nil)
// Stacktrie requires sorted insertion
sort.Sort(vals)
for _, kv := range vals {
if f.debugging {
fmt.Printf("{\"0x%x\" , \"0x%x\"} // stacktrie.Update\n", kv.k, kv.v)
}
trieB.Update(kv.k, kv.v)
}
rootB := trieB.Hash()
if _, err := trieB.Commit(); err != nil {
panic(err)
}
if rootA != rootB {
panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootB))
}
sumA := spongeA.sponge.Sum(nil)
sumB := spongeB.sponge.Sum(nil)
if !bytes.Equal(sumA, sumB) {
panic(fmt.Sprintf("sequence differ: (trie) %x != %x (stacktrie)", sumA, sumB))
}
return 1
}

11
tests/fuzzers/trie/trie-fuzzer.go
View file

@ -122,15 +122,22 @@ func Generate(input []byte) randTest {
return steps
}
// The function must return
// 1 if the fuzzer should increase priority of the
// given input during subsequent fuzzing (for example, the input is lexically
// correct and was parsed successfully);
// -1 if the input must not be added to corpus even if gives new coverage; and
// 0 otherwise
// other values are reserved for future use.
func Fuzz(input []byte) int {
program := Generate(input)
if len(program) == 0 {
return -1
return 0
}
if err := runRandTest(program); err != nil {
panic(err)
}
return 0
return 1
}
func runRandTest(rt randTest) error {

8
tests/init.go
View file

@ -141,7 +141,7 @@ var Forks = map[string]*params.ChainConfig{
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(5),
},
"YOLOv1": {
"YOLOv2": {
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
@ -151,9 +151,9 @@ var Forks = map[string]*params.ChainConfig{
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
YoloV1Block: big.NewInt(0),
YoloV2Block: big.NewInt(0),
},
// This specification is subject to change, but is for now identical to YOLOv1
// This specification is subject to change, but is for now identical to YOLOv2
// for cross-client testing purposes
"Berlin": {
ChainID: big.NewInt(1),
@ -165,7 +165,7 @@ var Forks = map[string]*params.ChainConfig{
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
YoloV1Block: big.NewInt(0),
YoloV2Block: big.NewInt(0),
},
}

12
tests/state_test_util.go
View file

@ -186,6 +186,16 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
context.GetHash = vmTestBlockHash
evm := vm.NewEVM(context, statedb, config, vmconfig)
if config.IsYoloV2(context.BlockNumber) {
statedb.AddAddressToAccessList(msg.From())
if dst := msg.To(); dst != nil {
statedb.AddAddressToAccessList(*dst)
// If it's a create-tx, the destination will be added inside evm.create
}
for _, addr := range evm.ActivePrecompiles() {
statedb.AddAddressToAccessList(addr)
}
}
gaspool := new(core.GasPool)
gaspool.AddGas(block.GasLimit())
snapshot := statedb.Snapshot()
@ -225,7 +235,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo
var snaps *snapshot.Tree
if snapshotter {
snaps = snapshot.New(db, sdb.TrieDB(), 1, root, false)
snaps = snapshot.New(db, sdb.TrieDB(), 1, root, false, false)
}
statedb, _ = state.New(root, sdb, snaps)
return snaps, statedb

39
trie/stacktrie.go
View file

@ -314,19 +314,22 @@ func (st *StackTrie) hash() {
panic(err)
}
case extNode:
st.children[0].hash()
h = newHasher(false)
defer returnHasherToPool(h)
h.tmp.Reset()
st.children[0].hash()
// This is also possible:
//sz := hexToCompactInPlace(st.key)
//n := [][]byte{
// st.key[:sz],
// st.children[0].val,
//}
n := [][]byte{
hexToCompact(st.key),
st.children[0].val,
var valuenode node
if len(st.children[0].val) < 32 {
valuenode = rawNode(st.children[0].val)
} else {
valuenode = hashNode(st.children[0].val)
}
n := struct {
Key []byte
Val node
}{
Key: hexToCompact(st.key),
Val: valuenode,
}
if err := rlp.Encode(&h.tmp, n); err != nil {
panic(err)
@ -406,6 +409,18 @@ func (st *StackTrie) Commit() (common.Hash, error) {
return common.Hash{}, ErrCommitDisabled
}
st.hash()
h := common.BytesToHash(st.val)
return h, nil
if len(st.val) != 32 {
// If the node's RLP isn't 32 bytes long, the node will not
// be hashed (and committed), and instead contain the rlp-encoding of the
// node. For the top level node, we need to force the hashing+commit.
ret := make([]byte, 32)
h := newHasher(false)
defer returnHasherToPool(h)
h.sha.Reset()
h.sha.Write(st.val)
h.sha.Read(ret)
st.db.Put(ret, st.val)
return common.BytesToHash(ret), nil
}
return common.BytesToHash(st.val), nil
}

49
trie/stacktrie_test.go
View file

@ -240,3 +240,52 @@ func TestDerivableList(t *testing.T) {
}
}
}
// TestUpdateSmallNodes tests a case where the leaves are small (both key and value),
// which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) {
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
kvs := []struct {
K string
V string
}{
{"63303030", "3041"}, // stacktrie.Update
{"65", "3000"}, // stacktrie.Update
}
for _, kv := range kvs {
nt.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
st.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
}
if nt.Hash() != st.Hash() {
t.Fatalf("error %x != %x", st.Hash(), nt.Hash())
}
}
// TestUpdateVariableKeys contains a case which stacktrie fails: when keys of different
// sizes are used, and the second one has the same prefix as the first, then the
// stacktrie fails, since it's unable to 'expand' on an already added leaf.
// For all practical purposes, this is fine, since keys are fixed-size length
// in account and storage tries.
//
// The test is marked as 'skipped', and exists just to have the behaviour documented.
// This case was found via fuzzing.
func TestUpdateVariableKeys(t *testing.T) {
t.SkipNow()
st := NewStackTrie(nil)
nt, _ := New(common.Hash{}, NewDatabase(memorydb.New()))
kvs := []struct {
K string
V string
}{
{"0x33303534636532393561313031676174", "303030"},
{"0x3330353463653239356131303167617430", "313131"},
}
for _, kv := range kvs {
nt.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
st.TryUpdate(common.FromHex(kv.K), common.FromHex(kv.V))
}
if nt.Hash() != st.Hash() {
t.Fatalf("error %x != %x", st.Hash(), nt.Hash())
}
}

36
trie/trie_test.go
View file

@ -853,6 +853,42 @@ func TestCommitSequenceStackTrie(t *testing.T) {
}
}
// TestCommitSequenceSmallRoot tests that a trie which is essentially only a
// small (<32 byte) shortnode with an included value is properly committed to a
// database.
// This case might not matter, since in practice, all keys are 32 bytes, which means
// that even a small trie which contains a leaf will have an extension making it
// not fit into 32 bytes, rlp-encoded. However, it's still the correct thing to do.
func TestCommitSequenceSmallRoot(t *testing.T) {
s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"}
db := NewDatabase(s)
trie, _ := New(common.Hash{}, db)
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(stackTrieSponge)
// Add a single small-element to the trie(s)
key := make([]byte, 5)
key[0] = 1
trie.TryUpdate(key, []byte{0x1})
stTrie.TryUpdate(key, []byte{0x1})
// Flush trie -> database
root, _ := trie.Commit(nil)
// Flush memdb -> disk (sponge)
db.Commit(root, false, nil)
// And flush stacktrie -> disk
stRoot, err := stTrie.Commit()
if err != nil {
t.Fatalf("Failed to commit stack trie %v", err)
}
if stRoot != root {
t.Fatalf("root wrong, got %x exp %x", stRoot, root)
}
fmt.Printf("root: %x\n", stRoot)
if got, exp := stackTrieSponge.sponge.Sum(nil), s.sponge.Sum(nil); !bytes.Equal(got, exp) {
t.Fatalf("test, disk write sequence wrong:\ngot %x exp %x\n", got, exp)
}
}
// BenchmarkCommitAfterHashFixedSize benchmarks the Commit (after Hash) of a fixed number of updates to a trie.
// This benchmark is meant to capture the difference on efficiency of small versus large changes. Typically,
// storage tries are small (a couple of entries), whereas the full post-block account trie update is large (a couple