update: merge v1.10.3

.github/CODEOWNERS

@@ -9,6 +9,7 @@ cmd/puppeth                     @karalabe
 consensus                       @karalabe
 core/                           @karalabe @holiman @rjl493456442
 eth/                            @karalabe @holiman @rjl493456442
+eth/catalyst/                   @gballet
 graphql/                        @gballet
 les/                            @zsfelfoldi @rjl493456442
 light/                          @zsfelfoldi @rjl493456442

.github/ISSUE_TEMPLATE/bug.md

@@ -26,3 +26,5 @@ Commit hash : (if `develop`)
 ````
 [backtrace]
 ````
+
+When submitting logs: please submit them as text and not screenshots.

Makefile

@@ -61,12 +61,11 @@ clean:
 # You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.
 
 devtools:
-	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
-	env GOBIN= go get -u github.com/kevinburke/go-bindata/go-bindata
-	env GOBIN= go get -u github.com/fjl/gencodec
-	env GOBIN= go get -u github.com/golang/protobuf/protoc-gen-go
+	env GOBIN= go install golang.org/x/tools/cmd/stringer@latest
+	env GOBIN= go install github.com/kevinburke/go-bindata/go-bindata@latest
+	env GOBIN= go install github.com/fjl/gencodec@latest
+	env GOBIN= go install github.com/golang/protobuf/protoc-gen-go@latest
 	env GOBIN= go install ./cmd/abigen
-	@type "npm" 2> /dev/null || echo 'Please install node.js and npm'
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 

README.md

@@ -4,7 +4,7 @@ Official Golang implementation of the Matic protocol (fork of Go Ethereum - http
 
 ## Building the source
 
-Building `bor` requires both a Go (version 1.13 or later) and a C compiler. You can install
+Building `bor` requires both a Go (version 1.14 or later) and a C compiler. You can install
 them using your favourite package manager. Once the dependencies are installed, run
 
 ```shell

accounts/abi/bind/base.go

@@ -54,6 +54,8 @@ type TransactOpts struct {
 	GasLimit uint64   // Gas limit to set for the transaction execution (0 = estimate)
 
 	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+
+	NoSend bool // Do all transact steps but do not send the transaction
 }
 
 // FilterOpts is the collection of options to fine tune filtering for events
@@ -260,6 +262,9 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	if err != nil {
 		return nil, err
 	}
+	if opts.NoSend {
+		return signedTx, nil
+	}
 	if err := c.transactor.SendTransaction(ensureContext(opts.Context), signedTx); err != nil {
 		return nil, err
 	}

accounts/accounts.go

@@ -114,7 +114,7 @@ type Wallet interface {
 	SignData(account Account, mimeType string, data []byte) ([]byte, error)
 
 	// SignDataWithPassphrase is identical to SignData, but also takes a password
-	// NOTE: there's an chance that an erroneous call might mistake the two strings, and
+	// NOTE: there's a chance that an erroneous call might mistake the two strings, and
 	// supply password in the mimetype field, or vice versa. Thus, an implementation
 	// should never echo the mimetype or return the mimetype in the error-response
 	SignDataWithPassphrase(account Account, passphrase, mimeType string, data []byte) ([]byte, error)
@@ -128,7 +128,7 @@ type Wallet interface {
 	// a password to decrypt the account, or a PIN code o verify the transaction),
 	// an AuthNeededError instance will be returned, containing infos for the user
 	// about which fields or actions are needed. The user may retry by providing
-	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the needed details via SignTextWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
 	//
 	// This method should return the signature in 'canonical' format, with v 0 or 1

accounts/external/backend.go

@@ -212,6 +212,20 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio
 		To:       to,
 		From:     common.NewMixedcaseAddress(account.Address),
 	}
+	// We should request the default chain id that we're operating with
+	// (the chain we're executing on)
+	if chainID != nil {
+		args.ChainID = (*hexutil.Big)(chainID)
+	}
+	// However, if the user asked for a particular chain id, then we should
+	// use that instead.
+	if tx.Type() != types.LegacyTxType && tx.ChainId() != nil {
+		args.ChainID = (*hexutil.Big)(tx.ChainId())
+	}
+	if tx.Type() == types.AccessListTxType {
+		accessList := tx.AccessList()
+		args.AccessList = &accessList
+	}
 	var res signTransactionResult
 	if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
 		return nil, err

accounts/url.go

@@ -64,7 +64,7 @@ func (u URL) String() string {
 func (u URL) TerminalString() string {
 	url := u.String()
 	if len(url) > 32 {
-		return url[:31] + "…"
+		return url[:31] + ".."
 	}
 	return url
 }

accounts/usbwallet/ledger.go

@@ -52,8 +52,10 @@ const (
 	ledgerOpRetrieveAddress  ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path
 	ledgerOpSignTransaction  ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters
 	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
+	ledgerOpSignTypedMessage ledgerOpcode = 0x0c // Signs an Ethereum message following the EIP 712 specification
 
 	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
+	ledgerP1InitTypedMessageData    ledgerParam1 = 0x00 // First chunk of Typed Message data
 	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
 	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
 	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
@@ -170,6 +172,24 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio
 	return w.ledgerSign(path, tx, chainID)
 }
 
+// SignTypedMessage implements usbwallet.driver, sending the message to the Ledger and
+// waiting for the user to sign or deny the transaction.
+//
+// Note: this was introduced in the ledger 1.5.0 firmware
+func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash []byte, messageHash []byte) ([]byte, error) {
+	// If the Ethereum app doesn't run, abort
+	if w.offline() {
+		return nil, accounts.ErrWalletClosed
+	}
+	// Ensure the wallet is capable of signing the given transaction
+	if w.version[0] < 1 && w.version[1] < 5 {
+		//lint:ignore ST1005 brand name displayed on the console
+		return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2])
+	}
+	// All infos gathered and metadata checks out, request signing
+	return w.ledgerSignTypedMessage(path, domainHash, messageHash)
+}
+
 // ledgerVersion retrieves the current version of the Ethereum wallet app running
 // on the Ledger wallet.
 //
@@ -367,6 +387,68 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 	return sender, signed, nil
 }
 
+// ledgerSignTypedMessage sends the transaction to the Ledger wallet, and waits for the user
+// to confirm or deny the transaction.
+//
+// The signing protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2                          | Lc  | Le
+//   ----+-----+----+-----------------------------+-----+---
+//    E0 | 0C  | 00 | implementation version : 00 | variable | variable
+//
+// Where the input is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+----------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//   domain hash                                      | 32 bytes
+//   message hash                                     | 32 bytes
+//
+//
+//
+// And the output data is:
+//
+//   Description | Length
+//   ------------+---------
+//   signature V | 1 byte
+//   signature R | 32 bytes
+//   signature S | 32 bytes
+func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHash []byte, messageHash []byte) ([]byte, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Create the 712 message
+	payload := append(path, domainHash...)
+	payload = append(payload, messageHash...)
+
+	// Send the request and wait for the response
+	var (
+		op    = ledgerP1InitTypedMessageData
+		reply []byte
+		err   error
+	)
+
+	// Send the message over, ensuring it's processed correctly
+	reply, err = w.ledgerExchange(ledgerOpSignTypedMessage, op, 0, payload)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Extract the Ethereum signature and do a sanity validation
+	if len(reply) != crypto.SignatureLength {
+		return nil, errors.New("reply lacks signature")
+	}
+	signature := append(reply[1:], reply[0])
+	return signature, nil
+}
+
 // ledgerExchange performs a data exchange with the Ledger wallet, sending it a
 // message and retrieving the response.
 //

accounts/usbwallet/trezor.go

@@ -185,6 +185,10 @@ func (w *trezorDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio
 	return w.trezorSign(path, tx, chainID)
 }
 
+func (w *trezorDriver) SignTypedMessage(path accounts.DerivationPath, domainHash []byte, messageHash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
 // trezorDerive sends a derivation request to the Trezor device and returns the
 // Ethereum address located on that path.
 func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, error) {

accounts/usbwallet/wallet.go

@@ -67,6 +67,8 @@ type driver interface {
 	// SignTx sends the transaction to the USB device and waits for the user to confirm
 	// or deny the transaction.
 	SignTx(path accounts.DerivationPath, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error)
+
+	SignTypedMessage(path accounts.DerivationPath, messageHash []byte, domainHash []byte) ([]byte, error)
 }
 
 // wallet represents the common functionality shared by all USB hardware
@@ -524,9 +526,48 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error)
 
 // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
 func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+
+	// Unless we are doing 712 signing, simply dispatch to signHash
+	if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) {
 		return w.signHash(account, crypto.Keccak256(data))
 	}
 
+	// dispatch to 712 signing if the mimetype is TypedData and the format matches
+	w.stateLock.RLock() // Comms have own mutex, this is for the state fields
+	defer w.stateLock.RUnlock()
+
+	// If the wallet is closed, abort
+	if w.device == nil {
+		return nil, accounts.ErrWalletClosed
+	}
+	// Make sure the requested account is contained within
+	path, ok := w.paths[account.Address]
+	if !ok {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// All infos gathered and metadata checks out, request signing
+	<-w.commsLock
+	defer func() { w.commsLock <- struct{}{} }()
+
+	// Ensure the device isn't screwed with while user confirmation is pending
+	// TODO(karalabe): remove if hotplug lands on Windows
+	w.hub.commsLock.Lock()
+	w.hub.commsPend++
+	w.hub.commsLock.Unlock()
+
+	defer func() {
+		w.hub.commsLock.Lock()
+		w.hub.commsPend--
+		w.hub.commsLock.Unlock()
+	}()
+	// Sign the transaction
+	signature, err := w.driver.SignTypedMessage(path, data[2:34], data[34:66])
+	if err != nil {
+		return nil, err
+	}
+	return signature, nil
+}
+
 // SignDataWithPassphrase implements accounts.Wallet, attempting to sign the given
 // data with the given account using passphrase as extra authentication.
 // Since USB wallets don't rely on passphrases, these are silently ignored.

appveyor.yml

@@ -1,41 +1,29 @@
-os: Visual Studio 2015
-
-# Clone directly into GOPATH.
-clone_folder: C:\gopath\src\github.com\ethereum\go-ethereum
+os: Visual Studio 2019
 clone_depth: 5
 version: "{branch}.{build}"
 environment:
-  global:
-    GO111MODULE: on
-    GOPATH: C:\gopath
-    CC: gcc.exe
   matrix:
+    # We use gcc from MSYS2 because it is the most recent compiler version available on
+    # AppVeyor. Note: gcc.exe only works properly if the corresponding bin/ directory is
+    # contained in PATH.
     - GETH_ARCH: amd64
-      MSYS2_ARCH: x86_64
-      MSYS2_BITS: 64
-      MSYSTEM: MINGW64
-      PATH: C:\msys64\mingw64\bin\;C:\Program Files (x86)\NSIS\;%PATH%
+      GETH_CC: C:\msys64\mingw64\bin\gcc.exe
+      PATH: C:\msys64\mingw64\bin;C:\Program Files (x86)\NSIS\;%PATH%
     - GETH_ARCH: 386
-      MSYS2_ARCH: i686
-      MSYS2_BITS: 32
-      MSYSTEM: MINGW32
-      PATH: C:\msys64\mingw32\bin\;C:\Program Files (x86)\NSIS\;%PATH%
+      GETH_CC: C:\msys64\mingw32\bin\gcc.exe
+      PATH: C:\msys64\mingw32\bin;C:\Program Files (x86)\NSIS\;%PATH%
 
 install:
-  - git submodule update --init
-  - rmdir C:\go /s /q
-  - appveyor DownloadFile https://dl.google.com/go/go1.16.windows-%GETH_ARCH%.zip
-  - 7z x go1.16.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - git submodule update --init --depth 1
   - go version
-  - gcc --version
+  - "%GETH_CC% --version"
 
 build_script:
-  - go run build\ci.go install -dlgo
+  - go run build\ci.go install -dlgo -arch %GETH_ARCH% -cc %GETH_CC%
 
 after_build:
-  - go run build\ci.go archive -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
-  - go run build\ci.go nsis -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
+  - go run build\ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
+  - go run build\ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
 
 test_script:
-  - set CGO_ENABLED=1
-  - go run build\ci.go test -coverage
+  - go run build\ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -coverage

build/checksums.txt

@@ -1,31 +1,33 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-7688063d55656105898f323d90a79a39c378d86fe89ae192eb3b7fc46347c95a  go1.16.src.tar.gz
-6000a9522975d116bf76044967d7e69e04e982e9625330d9a539a8b45395f9a8  go1.16.darwin-amd64.tar.gz
-ea435a1ac6d497b03e367fdfb74b33e961d813883468080f6e239b3b03bea6aa  go1.16.linux-386.tar.gz
-013a489ebb3e24ef3d915abe5b94c3286c070dfe0818d5bca8108f1d6e8440d2  go1.16.linux-amd64.tar.gz
-3770f7eb22d05e25fbee8fb53c2a4e897da043eb83c69b9a14f8d98562cd8098  go1.16.linux-arm64.tar.gz
-d1d9404b1dbd77afa2bdc70934e10fbfcf7d785c372efc29462bb7d83d0a32fd  go1.16.linux-armv6l.tar.gz
-481492a17d42193d471b93b7a06da3555331bd833b76336afc87be820c48933f  go1.16.windows-386.zip
-5cc88fa506b3d5c453c54c3ea218fc8dd05d7362ae1de15bb67986b72089ce93  go1.16.windows-amd64.zip
-d7d6c70b05a7c2f68b48aab5ab8cb5116b8444c9ddad131673b152e7cff7c726  go1.16.freebsd-386.tar.gz
-40b03216f6945fb6883a50604fc7f409a83f62171607229a9c598e701e684f8a  go1.16.freebsd-amd64.tar.gz
-27a1aaa988e930b7932ce459c8a63ad5b3333b3a06b016d87ff289f2a11aacd6  go1.16.linux-ppc64le.tar.gz
-be4c9e4e2cf058efc4e3eb013a760cb989ddc4362f111950c990d1c63b27ccbe  go1.16.linux-s390x.tar.gz
+b298d29de9236ca47a023e382313bcc2d2eed31dfa706b60a04103ce83a71a25  go1.16.3.src.tar.gz
+6bb1cf421f8abc2a9a4e39140b7397cdae6aca3e8d36dcff39a1a77f4f1170ac  go1.16.3.darwin-amd64.tar.gz
+f4e96bbcd5d2d1942f5b55d9e4ab19564da4fad192012f6d7b0b9b055ba4208f  go1.16.3.darwin-arm64.tar.gz
+48b2d1481db756c88c18b1f064dbfc3e265ce4a775a23177ca17e25d13a24c5d  go1.16.3.linux-386.tar.gz
+951a3c7c6ce4e56ad883f97d9db74d3d6d80d5fec77455c6ada6c1f7ac4776d2  go1.16.3.linux-amd64.tar.gz
+566b1d6f17d2bc4ad5f81486f0df44f3088c3ed47a3bec4099d8ed9939e90d5d  go1.16.3.linux-arm64.tar.gz
+0dae30385e3564a557dac7f12a63eedc73543e6da0f6017990e214ce8cc8797c  go1.16.3.linux-armv6l.tar.gz
+a3c16e1531bf9726f47911c4a9ed7cb665a6207a51c44f10ebad4db63b4bcc5a  go1.16.3.windows-386.zip
+a4400345135b36cb7942e52bbaf978b66814738b855eeff8de879a09fd99de7f  go1.16.3.windows-amd64.zip
+31ecd11d497684fa8b0f01ba784590c4c760943665fdc4fe0adaa1405c71736c  go1.16.3.freebsd-386.tar.gz
+ffbd920b309e62e807457b11d80e8c17fefe3ef6de423aaba4b1e270b2ca4c3d  go1.16.3.freebsd-amd64.tar.gz
+5eb046bbbbc7fe2591846a4303884cb5a01abb903e3e61e33459affe7874e811  go1.16.3.linux-ppc64le.tar.gz
+3e8bd7bde533a73fd6fa75b5288678ef397e76c198cfb26b8ae086035383b1cf  go1.16.3.linux-s390x.tar.gz
 
-d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35  golangci-lint-1.27.0-linux-armv7.tar.gz
-bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4  golangci-lint.exe-1.27.0-windows-amd64.zip
-bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4  golangci-lint-1.27.0-windows-amd64.zip
-0e2a57d6ba709440d3ed018ef1037465fa010ed02595829092860e5cf863042e  golangci-lint-1.27.0-freebsd-386.tar.gz
-90205fc42ab5ed0096413e790d88ac9b4ed60f4c47e576d13dc0660f7ed4b013  golangci-lint-1.27.0-linux-arm64.tar.gz
-8d345e4e88520e21c113d81978e89ad77fc5b13bfdf20e5bca86b83fc4261272  golangci-lint-1.27.0-linux-amd64.tar.gz
-cc619634a77f18dc73df2a0725be13116d64328dc35131ca1737a850d6f76a59  golangci-lint-1.27.0-freebsd-armv7.tar.gz
-fe683583cfc9eeec83e498c0d6159d87b5e1919dbe4b6c3b3913089642906069  golangci-lint-1.27.0-linux-s390x.tar.gz
-058f5579bee75bdaacbaf75b75e1369f7ad877fd8b3b145aed17a17545de913e  golangci-lint-1.27.0-freebsd-armv6.tar.gz
-38e1e3dadbe3f56ab62b4de82ee0b88e8fad966d8dfd740a26ef94c2edef9818  golangci-lint-1.27.0-linux-armv6.tar.gz
-071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095  golangci-lint.exe-1.27.0-windows-386.zip
-071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095  golangci-lint-1.27.0-windows-386.zip
-5f37e2b33914ecddb7cad38186ef4ec61d88172fc04f930fa0267c91151ff306  golangci-lint-1.27.0-linux-386.tar.gz
-4d94cfb51fdebeb205f1d5a349ac2b683c30591c5150708073c1c329e15965f0  golangci-lint-1.27.0-freebsd-amd64.tar.gz
-52572ba8ff07d5169c2365d3de3fec26dc55a97522094d13d1596199580fa281  golangci-lint-1.27.0-linux-ppc64le.tar.gz
-3fb1a1683a29c6c0a8cd76135f62b606fbdd538d5a7aeab94af1af70ffdc2fd4  golangci-lint-1.27.0-darwin-amd64.tar.gz
+7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c  golangci-lint-1.39.0-darwin-amd64.tar.gz
+574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064  golangci-lint-1.39.0-darwin-arm64.tar.gz
+6225f7014987324ab78e9b511f294e3f25be013728283c33918c67c8576d543e  golangci-lint-1.39.0-freebsd-386.tar.gz
+6b3e76e1e5eaf0159411c8e2727f8d533989d3bb19f10e9caa6e0b9619ee267d  golangci-lint-1.39.0-freebsd-amd64.tar.gz
+a301cacfff87ed9b00313d95278533c25a4527a06b040a17d969b4b7e1b8a90d  golangci-lint-1.39.0-freebsd-armv7.tar.gz
+25bfd96a29c3112f508d5e4fc860dbad7afce657233c343acfa20715717d51e7  golangci-lint-1.39.0-freebsd-armv6.tar.gz
+9687e4ff15545cfc722b0e46107a94195166a505023b48a316579af25ad09505  golangci-lint-1.39.0-linux-armv7.tar.gz
+a7fa7ab2bfc99cbe5e5bcbf5684f5a997f920afbbe2f253d2feb1001d5e3c8b3  golangci-lint-1.39.0-linux-armv6.tar.gz
+c8f9634115beddb4ed9129c1f7ecd4c97c99d07aeef33e3707234097eeb51b7b  golangci-lint-1.39.0-linux-mips64le.tar.gz
+d1234c213b74751f1af413302dde0e9a6d4d29aecef034af7abb07dc1b6e887f  golangci-lint-1.39.0-linux-arm64.tar.gz
+df25d9267168323b163147acb823ab0215a8a3bb6898a4a9320afdfedde66817  golangci-lint-1.39.0-linux-386.tar.gz
+1767e75fba357b7651b1a796d38453558f371c60af805505ec99e166908c04b5  golangci-lint-1.39.0-linux-ppc64le.tar.gz
+25fd75bf3186b3d930ecae10185689968fd18fd8fa6f9f555d6beb04348c20f6  golangci-lint-1.39.0-linux-s390x.tar.gz
+3a73aa7468087caa62673c8adea99b4e4dff846dc72707222db85f8679b40cbf  golangci-lint-1.39.0-linux-amd64.tar.gz
+578caceccf81739bda67dbfec52816709d03608c6878888ecdc0e186a094a41b  golangci-lint-1.39.0-linux-mips64.tar.gz
+494b66ba0e32c8ddf6c4f6b1d05729b110900f6017eda943057e43598c17d7a8  golangci-lint-1.39.0-windows-386.zip
+52ec2e13a3cbb47147244dff8cfc35103563deb76e0459133058086fc35fb2c7  golangci-lint-1.39.0-windows-amd64.zip

build/ci.go

@@ -152,7 +152,7 @@ var (
 	// This is the version of go that will be downloaded by
 	//
 	//     go run ci.go install -dlgo
-	dlgoVersion = "1.16"
+	dlgoVersion = "1.16.3"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -208,58 +208,25 @@ func doInstall(cmdline []string) {
 		cc   = flag.String("cc", "", "C compiler to cross build with")
 	)
 	flag.CommandLine.Parse(cmdline)
+
+	// Configure the toolchain.
+	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
+	if *dlgo {
+		csdb := build.MustLoadChecksums("build/checksums.txt")
+		tc.Root = build.DownloadGo(csdb, dlgoVersion)
+	}
+
+	// Configure the build.
 	env := build.Env()
-
-	// Check local Go version. People regularly open issues about compilation
-	// failure with outdated Go. This should save them the trouble.
-	if !strings.Contains(runtime.Version(), "devel") {
-		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
-		var minor int
-		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
-		if minor < 13 {
-			log.Println("You have Go version", runtime.Version())
-			log.Println("go-ethereum requires at least Go version 1.13 and cannot")
-			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
-			os.Exit(1)
-		}
-	}
-
-	// Choose which go command we're going to use.
-	var gobuild *exec.Cmd
-	if !*dlgo {
-		// Default behavior: use the go version which runs ci.go right now.
-		gobuild = goTool("build")
-	} else {
-		// Download of Go requested. This is for build environments where the
-		// installed version is too old and cannot be upgraded easily.
-		cachedir := filepath.Join("build", "cache")
-		goroot := downloadGo(runtime.GOARCH, runtime.GOOS, cachedir)
-		gobuild = localGoTool(goroot, "build")
-	}
-
-	// Configure environment for cross build.
-	if *arch != "" || *arch != runtime.GOARCH {
-		gobuild.Env = append(gobuild.Env, "CGO_ENABLED=1")
-		gobuild.Env = append(gobuild.Env, "GOARCH="+*arch)
-	}
-
-	// Configure C compiler.
-	if *cc != "" {
-		gobuild.Env = append(gobuild.Env, "CC="+*cc)
-	} else if os.Getenv("CC") != "" {
-		gobuild.Env = append(gobuild.Env, "CC="+os.Getenv("CC"))
-	}
+	gobuild := tc.Go("build", buildFlags(env)...)
 
 	// arm64 CI builders are memory-constrained and can't handle concurrent builds,
 	// better disable it. This check isn't the best, it should probably
 	// check for something in env instead.
-	if runtime.GOARCH == "arm64" {
+	if env.CI && runtime.GOARCH == "arm64" {
 		gobuild.Args = append(gobuild.Args, "-p", "1")
 	}
 
-	// Put the default settings in.
-	gobuild.Args = append(gobuild.Args, buildFlags(env)...)
-
 	// We use -trimpath to avoid leaking local paths into the built executables.
 	gobuild.Args = append(gobuild.Args, "-trimpath")
 
@@ -301,53 +268,30 @@ func buildFlags(env build.Environment) (flags []string) {
 	return flags
 }
 
-// goTool returns the go tool. This uses the Go version which runs ci.go.
-func goTool(subcmd string, args ...string) *exec.Cmd {
-	cmd := build.GoTool(subcmd, args...)
-	goToolSetEnv(cmd)
-	return cmd
-}
-
-// localGoTool returns the go tool from the given GOROOT.
-func localGoTool(goroot string, subcmd string, args ...string) *exec.Cmd {
-	gotool := filepath.Join(goroot, "bin", "go")
-	cmd := exec.Command(gotool, subcmd)
-	goToolSetEnv(cmd)
-	cmd.Env = append(cmd.Env, "GOROOT="+goroot)
-	cmd.Args = append(cmd.Args, args...)
-	return cmd
-}
-
-// goToolSetEnv forwards the build environment to the go tool.
-func goToolSetEnv(cmd *exec.Cmd) {
-	cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
-	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOBIN=") || strings.HasPrefix(e, "CC=") {
-			continue
-		}
-		cmd.Env = append(cmd.Env, e)
-	}
-}
-
 // Running The Tests
 //
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
-	verbose := flag.Bool("v", false, "Whether to log verbosely")
+	var (
+		dlgo     = flag.Bool("dlgo", false, "Download Go and build with it")
+		arch     = flag.String("arch", "", "Run tests for given architecture")
+		cc       = flag.String("cc", "", "Sets C compiler binary")
+		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
+		verbose  = flag.Bool("v", false, "Whether to log verbosely")
+	)
 	flag.CommandLine.Parse(cmdline)
-	env := build.Env()
 
-	packages := []string{"./..."}
-	if len(flag.CommandLine.Args()) > 0 {
-		packages = flag.CommandLine.Args()
+	// Configure the toolchain.
+	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
+	if *dlgo {
+		csdb := build.MustLoadChecksums("build/checksums.txt")
+		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
+	gotest := tc.Go("test")
 
-	// Run the actual tests.
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest := goTool("test", buildFlags(env)...)
 	gotest.Args = append(gotest.Args, "-p", "1")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
@@ -356,6 +300,10 @@ func doTest(cmdline []string) {
 		gotest.Args = append(gotest.Args, "-v")
 	}
 
+	packages := []string{"./..."}
+	if len(flag.CommandLine.Args()) > 0 {
+		packages = flag.CommandLine.Args()
+	}
 	gotest.Args = append(gotest.Args, packages...)
 	build.MustRun(gotest)
 }
@@ -379,7 +327,7 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	const version = "1.27.0"
+	const version = "1.39.0"
 
 	csdb := build.MustLoadChecksums("build/checksums.txt")
 	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)
@@ -416,7 +364,6 @@ func doArchive(cmdline []string) {
 
 	var (
 		env      = build.Env()
-
 		basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit))
 		geth     = "geth-" + basegeth + ext
 		alltools = "geth-alltools-" + basegeth + ext
@@ -492,15 +439,15 @@ func archiveUpload(archive string, blobstore string, signer string, signifyVar s
 // skips archiving for some build configurations.
 func maybeSkipArchive(env build.Environment) {
 	if env.IsPullRequest {
-		log.Printf("skipping because this is a PR build")
+		log.Printf("skipping archive creation because this is a PR build")
 		os.Exit(0)
 	}
 	if env.IsCronJob {
-		log.Printf("skipping because this is a cron job")
+		log.Printf("skipping archive creation because this is a cron job")
 		os.Exit(0)
 	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
-		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
+		log.Printf("skipping archive creation because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
 	}
 }
@@ -518,6 +465,7 @@ func doDebianSource(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 	*workdir = makeWorkdir(*workdir)
 	env := build.Env()
+	tc := new(build.GoToolchain)
 	maybeSkipArchive(env)
 
 	// Import the signing key.
@@ -531,12 +479,12 @@ func doDebianSource(cmdline []string) {
 	gobundle := downloadGoSources(*cachedir)
 
 	// Download all the dependencies needed to build the sources and run the ci script
-	srcdepfetch := goTool("mod", "download")
-	srcdepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath"))
+	srcdepfetch := tc.Go("mod", "download")
+	srcdepfetch.Env = append(srcdepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath"))
 	build.MustRun(srcdepfetch)
 
-	cidepfetch := goTool("run", "./build/ci.go")
-	cidepfetch.Env = append(os.Environ(), "GOPATH="+filepath.Join(*workdir, "modgopath"))
+	cidepfetch := tc.Go("run", "./build/ci.go")
+	cidepfetch.Env = append(cidepfetch.Env, "GOPATH="+filepath.Join(*workdir, "modgopath"))
 	cidepfetch.Run() // Command fails, don't care, we only need the deps to start it
 
 	// Create Debian packages and upload them.
@@ -592,41 +540,6 @@ func downloadGoSources(cachedir string) string {
 	return dst
 }
 
-// downloadGo downloads the Go binary distribution and unpacks it into a temporary
-// directory. It returns the GOROOT of the unpacked toolchain.
-func downloadGo(goarch, goos, cachedir string) string {
-	if goarch == "arm" {
-		goarch = "armv6l"
-	}
-
-	csdb := build.MustLoadChecksums("build/checksums.txt")
-	file := fmt.Sprintf("go%s.%s-%s", dlgoVersion, goos, goarch)
-	if goos == "windows" {
-		file += ".zip"
-	} else {
-		file += ".tar.gz"
-	}
-	url := "https://golang.org/dl/" + file
-	dst := filepath.Join(cachedir, file)
-	if err := csdb.DownloadFile(url, dst); err != nil {
-		log.Fatal(err)
-	}
-
-	ucache, err := os.UserCacheDir()
-	if err != nil {
-		log.Fatal(err)
-	}
-	godir := filepath.Join(ucache, fmt.Sprintf("geth-go-%s-%s-%s", dlgoVersion, goos, goarch))
-	if err := build.ExtractArchive(dst, godir); err != nil {
-		log.Fatal(err)
-	}
-	goroot, err := filepath.Abs(filepath.Join(godir, "go"))
-	if err != nil {
-		log.Fatal(err)
-	}
-	return goroot
-}
-
 func ppaUpload(workdir, ppa, sshUser string, files []string) {
 	p := strings.Split(ppa, "/")
 	if len(p) != 2 {
@@ -901,13 +814,23 @@ func doAndroidArchive(cmdline []string) {
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
+	tc := new(build.GoToolchain)
 
 	// Sanity check that the SDK and NDK are installed and set
 	if os.Getenv("ANDROID_HOME") == "" {
 		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
 	}
+
+	// Build gomobile.
+	install := tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest")
+	install.Env = append(install.Env)
+	build.MustRun(install)
+
+	// Ensure all dependencies are available. This is required to make
+	// gomobile bind work because it expects go.sum to contain all checksums.
+	build.MustRun(tc.Go("mod", "download"))
+
 	// Build the Android archive and Maven resources
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {
@@ -1027,10 +950,16 @@ func doXCodeFramework(cmdline []string) {
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
+	tc := new(build.GoToolchain)
+
+	// Build gomobile.
+	build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest"))
+
+	// Ensure all dependencies are available. This is required to make
+	// gomobile bind work because it expects go.sum to contain all checksums.
+	build.MustRun(tc.Go("mod", "download"))
 
 	// Build the iOS XCode framework
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
-	build.MustRun(gomobileTool("init"))
 	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
 
 	if *local {
@@ -1039,17 +968,17 @@ func doXCodeFramework(cmdline []string) {
 		build.MustRun(bind)
 		return
 	}
+
+	// Create the archive.
+	maybeSkipArchive(env)
 	archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit))
-	if err := os.Mkdir(archive, os.ModePerm); err != nil {
+	if err := os.MkdirAll(archive, 0755); err != nil {
 		log.Fatal(err)
 	}
 	bind.Dir, _ = filepath.Abs(archive)
 	build.MustRun(bind)
 	build.MustRunCommand("tar", "-zcvf", archive+".tar.gz", archive)
 
-	// Skip CocoaPods deploy and Azure upload for PR builds
-	maybeSkipArchive(env)
-
 	// Sign and upload the framework to Azure
 	if err := archiveUpload(archive+".tar.gz", *upload, *signer, *signify); err != nil {
 		log.Fatal(err)
@@ -1115,10 +1044,10 @@ func doXgo(cmdline []string) {
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
+	var tc build.GoToolchain
 
 	// Make sure xgo is available for cross compilation
-	gogetxgo := goTool("get", "github.com/karalabe/xgo")
-	build.MustRun(gogetxgo)
+	build.MustRun(tc.Install(GOBIN, "github.com/karalabe/xgo@latest"))
 
 	// If all tools building is requested, build everything the builder wants
 	args := append(buildFlags(env), flag.Args()...)
@@ -1129,27 +1058,23 @@ func doXgo(cmdline []string) {
 			if strings.HasPrefix(res, GOBIN) {
 				// Binary tool found, cross build it explicitly
 				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
-				xgo := xgoTool(args)
-				build.MustRun(xgo)
+				build.MustRun(xgoTool(args))
 				args = args[:len(args)-1]
 			}
 		}
 		return
 	}
-	// Otherwise xxecute the explicit cross compilation
+
+	// Otherwise execute the explicit cross compilation
 	path := args[len(args)-1]
 	args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
-
-	xgo := xgoTool(args)
-	build.MustRun(xgo)
+	build.MustRun(xgoTool(args))
 }
 
 func xgoTool(args []string) *exec.Cmd {
 	cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...)
 	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, []string{
-		"GOBIN=" + GOBIN,
-	}...)
+	cmd.Env = append(cmd.Env, []string{"GOBIN=" + GOBIN}...)
 	return cmd
 }
 

build/nsis.envvarupdate.nsh

@@ -43,7 +43,7 @@
   !ifndef Un${StrFuncName}_INCLUDED
     ${Un${StrFuncName}}
   !endif
-  !define un.${StrFuncName} "${Un${StrFuncName}}"
+  !define un.${StrFuncName} '${Un${StrFuncName}}'
 !macroend
 
 !insertmacro _IncludeStrFunction StrTok

cmd/clef/README.md

@@ -919,4 +919,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
-| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|
+| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/devp2p/README.md

@@ -30,6 +30,29 @@ Run `devp2p dns to-route53 <directory>` to publish a tree to Amazon Route53.
 
 You can find more information about these commands in the [DNS Discovery Setup Guide][dns-tutorial].
 
+### Node Set Utilities
+
+There are several commands for working with JSON node set files. These files are generated
+by the discovery crawlers and DNS client commands. Node sets also used as the input of the
+DNS deployer commands.
+
+Run `devp2p nodeset info <nodes.json>` to display statistics of a node set.
+
+Run `devp2p nodeset filter <nodes.json> <filter flags...>` to write a new, filtered node
+set to standard output. The following filters are supported:
+
+- `-limit <N>` limits the output set to N entries, taking the top N nodes by score
+- `-ip <CIDR>` filters nodes by IP subnet
+- `-min-age <duration>` filters nodes by 'first seen' time
+- `-eth-network <mainnet/rinkeby/goerli/ropsten>` filters nodes by "eth" ENR entry
+- `-les-server` filters nodes by LES server support
+- `-snap` filters nodes by snap protocol support
+
+For example, given a node set in `nodes.json`, you could create a filtered set containing
+up to 20 eth mainnet nodes which also support snap sync using this command:
+
+    devp2p nodeset filter nodes.json -eth-network mainnet -snap -limit 20
+
 ### Discovery v4 Utilities
 
 The `devp2p discv4 ...` command family deals with the [Node Discovery v4][discv4]

cmd/devp2p/dns_cloudflare.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -79,7 +80,7 @@ func (c *cloudflareClient) checkZone(name string) error {
 		c.zoneID = id
 	}
 	log.Info(fmt.Sprintf("Checking Permissions on zone %s", c.zoneID))
-	zone, err := c.ZoneDetails(c.zoneID)
+	zone, err := c.ZoneDetails(context.Background(), c.zoneID)
 	if err != nil {
 		return err
 	}
@@ -112,7 +113,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 	records = lrecords
 
 	log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name))
-	entries, err := c.DNSRecords(c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
+	entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
 	if err != nil {
 		return err
 	}
@@ -134,14 +135,15 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 			if path != name {
 				ttl = treeNodeTTL // Max TTL permitted by Cloudflare
 			}
-			_, err = c.CreateDNSRecord(c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl})
+			record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}
+			_, err = c.CreateDNSRecord(context.Background(), c.zoneID, record)
 		} else if old.Content != val {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val))
 			old.Content = val
-			err = c.UpdateDNSRecord(c.zoneID, old.ID, old)
+			err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old)
 		} else {
-			log.Info(fmt.Sprintf("Skipping %s = %q", path, val))
+			log.Debug(fmt.Sprintf("Skipping %s = %q", path, val))
 		}
 		if err != nil {
 			return fmt.Errorf("failed to publish %s: %v", path, err)
@@ -155,7 +157,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 		}
 		// Stale entry, nuke it.
 		log.Info(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
-		if err := c.DeleteDNSRecord(c.zoneID, entry.ID); err != nil {
+		if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil {
 			return fmt.Errorf("failed to delete %s: %v", path, err)
 		}
 	}

cmd/devp2p/dns_route53.go

@@ -17,16 +17,19 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/route53"
+	"github.com/aws/aws-sdk-go-v2/service/route53/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/dnsdisc"
 	"gopkg.in/urfave/cli.v1"
@@ -38,6 +41,7 @@ const (
 	// https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets
 	route53ChangeSizeLimit  = 32000
 	route53ChangeCountLimit = 1000
+	maxRetryLimit           = 60
 )
 
 var (
@@ -55,10 +59,15 @@ var (
 		Name:  "zone-id",
 		Usage: "Route53 Zone ID",
 	}
+	route53RegionFlag = cli.StringFlag{
+		Name:  "aws-region",
+		Usage: "AWS Region",
+		Value: "eu-central-1",
+	}
 )
 
 type route53Client struct {
-	api    *route53.Route53
+	api    *route53.Client
 	zoneID string
 }
 
@@ -72,15 +81,16 @@ func newRoute53Client(ctx *cli.Context) *route53Client {
 	akey := ctx.String(route53AccessKeyFlag.Name)
 	asec := ctx.String(route53AccessSecretFlag.Name)
 	if akey == "" || asec == "" {
-		exit(fmt.Errorf("need Route53 Access Key ID and secret proceed"))
+		exit(fmt.Errorf("need Route53 Access Key ID and secret to proceed"))
 	}
-	config := &aws.Config{Credentials: credentials.NewStaticCredentials(akey, asec, "")}
-	session, err := session.NewSession(config)
+	creds := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(akey, asec, ""))
+	cfg, err := config.LoadDefaultConfig(context.Background(), config.WithCredentialsProvider(creds))
 	if err != nil {
-		exit(fmt.Errorf("can't create AWS session: %v", err))
+		exit(fmt.Errorf("can't initialize AWS configuration: %v", err))
 	}
+	cfg.Region = ctx.String(route53RegionFlag.Name)
 	return &route53Client{
-		api:    route53.New(session),
+		api:    route53.NewFromConfig(cfg),
 		zoneID: ctx.String(route53ZoneIDFlag.Name),
 	}
 }
@@ -97,31 +107,74 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error {
 		return err
 	}
 	log.Info(fmt.Sprintf("Found %d TXT records", len(existing)))
-
 	records := t.ToTXT(name)
 	changes := c.computeChanges(name, records, existing)
+
+	// Submit to API.
+	comment := fmt.Sprintf("enrtree update of %s at seq %d", name, t.Seq())
+	return c.submitChanges(changes, comment)
+}
+
+// deleteDomain removes all TXT records of the given domain.
+func (c *route53Client) deleteDomain(name string) error {
+	if err := c.checkZone(name); err != nil {
+		return err
+	}
+
+	// Compute DNS changes.
+	existing, err := c.collectRecords(name)
+	if err != nil {
+		return err
+	}
+	log.Info(fmt.Sprintf("Found %d TXT records", len(existing)))
+	changes := makeDeletionChanges(existing, nil)
+
+	// Submit to API.
+	comment := "enrtree delete of " + name
+	return c.submitChanges(changes, comment)
+}
+
+// submitChanges submits the given DNS changes to Route53.
+func (c *route53Client) submitChanges(changes []types.Change, comment string) error {
 	if len(changes) == 0 {
 		log.Info("No DNS changes needed")
 		return nil
 	}
 
-	// Submit change batches.
+	var err error
 	batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit)
+	changesToCheck := make([]*route53.ChangeResourceRecordSetsOutput, len(batches))
 	for i, changes := range batches {
 		log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes)))
-		batch := new(route53.ChangeBatch)
-		batch.SetChanges(changes)
-		batch.SetComment(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq()))
+		batch := &types.ChangeBatch{
+			Changes: changes,
+			Comment: aws.String(fmt.Sprintf("%s (%d/%d)", comment, i+1, len(batches))),
+		}
 		req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch}
-		resp, err := c.api.ChangeResourceRecordSets(req)
+		changesToCheck[i], err = c.api.ChangeResourceRecordSets(context.TODO(), req)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Wait for all change batches to propagate.
+	for _, change := range changesToCheck {
+		log.Info(fmt.Sprintf("Waiting for change request %s", *change.ChangeInfo.Id))
+		wreq := &route53.GetChangeInput{Id: change.ChangeInfo.Id}
+		var count int
+		for {
+			wresp, err := c.api.GetChange(context.TODO(), wreq)
 			if err != nil {
 				return err
 			}
 
-		log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id))
-		wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id}
-		if err := c.api.WaitUntilResourceRecordSetsChanged(wreq); err != nil {
-			return err
+			count++
+
+			if wresp.ChangeInfo.Status == types.ChangeStatusInsync || count >= maxRetryLimit {
+				break
+			}
+
+			time.Sleep(30 * time.Second)
 		}
 	}
 	return nil
@@ -140,7 +193,7 @@ func (c *route53Client) findZoneID(name string) (string, error) {
 	log.Info(fmt.Sprintf("Finding Route53 Zone ID for %s", name))
 	var req route53.ListHostedZonesByNameInput
 	for {
-		resp, err := c.api.ListHostedZonesByName(&req)
+		resp, err := c.api.ListHostedZonesByName(context.TODO(), &req)
 		if err != nil {
 			return "", err
 		}
@@ -149,7 +202,7 @@ func (c *route53Client) findZoneID(name string) (string, error) {
 				return *zone.Id, nil
 			}
 		}
-		if !*resp.IsTruncated {
+		if !resp.IsTruncated {
 			break
 		}
 		req.DNSName = resp.NextDNSName
@@ -158,8 +211,9 @@ func (c *route53Client) findZoneID(name string) (string, error) {
 	return "", errors.New("can't find zone ID for " + name)
 }
 
-// computeChanges creates DNS changes for the given record.
-func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []*route53.Change {
+// computeChanges creates DNS changes for the given set of DNS discovery records.
+// The 'existing' arg is the set of records that already exist on Route53.
+func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []types.Change {
 	// Convert all names to lowercase.
 	lrecords := make(map[string]string, len(records))
 	for name, r := range records {
@@ -167,58 +221,70 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e
 	}
 	records = lrecords
 
-	var changes []*route53.Change
-	for path, val := range records {
+	var changes []types.Change
+	for path, newValue := range records {
+		prevRecords, exists := existing[path]
+		prevValue := strings.Join(prevRecords.values, "")
+
+		// prevValue contains quoted strings, encode newValue to compare.
+		newValue = splitTXT(newValue)
+
+		// Assign TTL.
 		ttl := int64(rootTTL)
 		if path != name {
 			ttl = int64(treeNodeTTL)
 		}
 
-		prevRecords, exists := existing[path]
-		prevValue := strings.Join(prevRecords.values, "")
 		if !exists {
 			// Entry is unknown, push a new one
-			log.Info(fmt.Sprintf("Creating %s = %q", path, val))
-			changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val)))
-		} else if prevValue != val || prevRecords.ttl != ttl {
+			log.Info(fmt.Sprintf("Creating %s = %s", path, newValue))
+			changes = append(changes, newTXTChange("CREATE", path, ttl, newValue))
+		} else if prevValue != newValue || prevRecords.ttl != ttl {
 			// Entry already exists, only change its content.
-			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val))
-			changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val)))
+			log.Info(fmt.Sprintf("Updating %s from %s to %s", path, prevValue, newValue))
+			changes = append(changes, newTXTChange("UPSERT", path, ttl, newValue))
 		} else {
-			log.Info(fmt.Sprintf("Skipping %s = %q", path, val))
+			log.Debug(fmt.Sprintf("Skipping %s = %s", path, newValue))
 		}
 	}
 
 	// Iterate over the old records and delete anything stale.
-	for path, set := range existing {
-		if _, ok := records[path]; ok {
-			continue
-		}
-		// Stale entry, nuke it.
-		log.Info(fmt.Sprintf("Deleting %s = %q", path, strings.Join(set.values, "")))
-		changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...))
-	}
+	changes = append(changes, makeDeletionChanges(existing, records)...)
 
+	// Ensure changes are in the correct order.
 	sortChanges(changes)
 	return changes
 }
 
+// makeDeletionChanges creates record changes which delete all records not contained in 'keep'.
+func makeDeletionChanges(records map[string]recordSet, keep map[string]string) []types.Change {
+	var changes []types.Change
+	for path, set := range records {
+		if _, ok := keep[path]; ok {
+			continue
+		}
+		log.Info(fmt.Sprintf("Deleting %s = %s", path, strings.Join(set.values, "")))
+		changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...))
+	}
+	return changes
+}
+
 // sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order.
-func sortChanges(changes []*route53.Change) {
+func sortChanges(changes []types.Change) {
 	score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3}
 	sort.Slice(changes, func(i, j int) bool {
-		if *changes[i].Action == *changes[j].Action {
+		if changes[i].Action == changes[j].Action {
 			return *changes[i].ResourceRecordSet.Name < *changes[j].ResourceRecordSet.Name
 		}
-		return score[*changes[i].Action] < score[*changes[j].Action]
+		return score[string(changes[i].Action)] < score[string(changes[j].Action)]
 	})
 }
 
 // splitChanges splits up DNS changes such that each change batch
 // is smaller than the given RDATA limit.
-func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*route53.Change {
+func splitChanges(changes []types.Change, sizeLimit, countLimit int) [][]types.Change {
 	var (
-		batches    [][]*route53.Change
+		batches    [][]types.Change
 		batchSize  int
 		batchCount int
 	)
@@ -241,7 +307,7 @@ func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*rou
 }
 
 // changeSize returns the RDATA size of a DNS change.
-func changeSize(ch *route53.Change) int {
+func changeSize(ch types.Change) int {
 	size := 0
 	for _, rr := range ch.ResourceRecordSet.ResourceRecords {
 		if rr.Value != nil {
@@ -251,8 +317,8 @@ func changeSize(ch *route53.Change) int {
 	return size
 }
 
-func changeCount(ch *route53.Change) int {
-	if *ch.Action == "UPSERT" {
+func changeCount(ch types.Change) int {
+	if ch.Action == types.ChangeActionUpsert {
 		return 2
 	}
 	return 1
@@ -260,13 +326,17 @@ func changeCount(ch *route53.Change) int {
 
 // collectRecords collects all TXT records below the given name.
 func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) {
-	log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID))
 	var req route53.ListResourceRecordSetsInput
-	req.SetHostedZoneId(c.zoneID)
+	req.HostedZoneId = &c.zoneID
 	existing := make(map[string]recordSet)
-	err := c.api.ListResourceRecordSetsPages(&req, func(resp *route53.ListResourceRecordSetsOutput, last bool) bool {
+	for page := 0; ; page++ {
+		log.Info("Loading existing TXT records", "name", name, "zone", c.zoneID, "page", page)
+		resp, err := c.api.ListResourceRecordSets(context.TODO(), &req)
+		if err != nil {
+			return existing, err
+		}
 		for _, set := range resp.ResourceRecordSets {
-			if !isSubdomain(*set.Name, name) || *set.Type != "TXT" {
+			if !isSubdomain(*set.Name, name) || set.Type != types.RRTypeTxt {
 				continue
 			}
 			s := recordSet{ttl: *set.TTL}
@@ -276,28 +346,44 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error
 			name := strings.TrimSuffix(*set.Name, ".")
 			existing[name] = s
 		}
-		return true
-	})
-	return existing, err
+
+		if !resp.IsTruncated {
+			break
+		}
+		// Set the cursor to the next batch. From the AWS docs:
+		//
+		// To display the next page of results, get the values of NextRecordName,
+		// NextRecordType, and NextRecordIdentifier (if any) from the response. Then submit
+		// another ListResourceRecordSets request, and specify those values for
+		// StartRecordName, StartRecordType, and StartRecordIdentifier.
+		req.StartRecordIdentifier = resp.NextRecordIdentifier
+		req.StartRecordName = resp.NextRecordName
+		req.StartRecordType = resp.NextRecordType
+	}
+
+	return existing, nil
 }
 
 // newTXTChange creates a change to a TXT record.
-func newTXTChange(action, name string, ttl int64, values ...string) *route53.Change {
-	var c route53.Change
-	var r route53.ResourceRecordSet
-	var rrs []*route53.ResourceRecord
+func newTXTChange(action, name string, ttl int64, values ...string) types.Change {
+	r := types.ResourceRecordSet{
+		Type: types.RRTypeTxt,
+		Name: &name,
+		TTL:  &ttl,
+	}
+	var rrs []types.ResourceRecord
 	for _, val := range values {
-		rr := new(route53.ResourceRecord)
-		rr.SetValue(val)
+		var rr types.ResourceRecord
+		rr.Value = aws.String(val)
 		rrs = append(rrs, rr)
 	}
-	r.SetType("TXT")
-	r.SetName(name)
-	r.SetTTL(ttl)
-	r.SetResourceRecords(rrs)
-	c.SetAction(action)
-	c.SetResourceRecordSet(&r)
-	return &c
+
+	r.ResourceRecords = rrs
+
+	return types.Change{
+		Action:            types.ChangeAction(action),
+		ResourceRecordSet: &r,
+	}
 }
 
 // isSubdomain returns true if name is a subdomain of domain.

cmd/devp2p/dns_route53_test.go

@@ -20,7 +20,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/aws/aws-sdk-go-v2/service/route53/types"
 )
 
 // This test checks that computeChanges/splitChanges create DNS changes in
@@ -43,93 +43,93 @@ func TestRoute53ChangeSort(t *testing.T) {
 		"MHTDO6TMUBRIA2XWG5LUDACK24.n": "enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o",
 	}
 
-	wantChanges := []*route53.Change{
+	wantChanges := []types.Change{
 		{
-			Action: sp("CREATE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "CREATE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("2xs2367yhaxjfglzhvawlqd4zy.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("CREATE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "CREATE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("c7hrfpf3blgf3yr4dy5kx3smbe.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("CREATE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "CREATE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("h4fht4b454p6uxfd7jcyq5pwdy.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("CREATE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "CREATE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("jwxydbpxywg6fx3gmdibfa6cj4.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("CREATE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "CREATE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("mhtdo6tmubria2xwg5ludack24.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("UPSERT"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "UPSERT",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`),
 				}},
 				TTL:  ip(rootTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("DELETE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "DELETE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("2kfjogvxdqtxxugbh7gs7naaai.n"),
-				ResourceRecords: []*route53.ResourceRecord{
+				ResourceRecords: []types.ResourceRecord{
 					{Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)},
 				},
 				TTL:  ip(3333),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 		{
-			Action: sp("DELETE"),
-			ResourceRecordSet: &route53.ResourceRecordSet{
+			Action: "DELETE",
+			ResourceRecordSet: &types.ResourceRecordSet{
 				Name: sp("fdxn3sn67na5dka4j2gok7bvqi.n"),
-				ResourceRecords: []*route53.ResourceRecord{{
+				ResourceRecords: []types.ResourceRecord{{
 					Value: sp(`"enrtree-branch:"`),
 				}},
 				TTL:  ip(treeNodeTTL),
-				Type: sp("TXT"),
+				Type: "TXT",
 			},
 		},
 	}
@@ -141,7 +141,7 @@ func TestRoute53ChangeSort(t *testing.T) {
 	}
 
 	// Check splitting according to size.
-	wantSplit := [][]*route53.Change{
+	wantSplit := [][]types.Change{
 		wantChanges[:4],
 		wantChanges[4:6],
 		wantChanges[6:],
@@ -152,7 +152,7 @@ func TestRoute53ChangeSort(t *testing.T) {
 	}
 
 	// Check splitting according to count.
-	wantSplit = [][]*route53.Change{
+	wantSplit = [][]types.Change{
 		wantChanges[:5],
 		wantChanges[5:],
 	}
@@ -162,5 +162,29 @@ func TestRoute53ChangeSort(t *testing.T) {
 	}
 }
 
+// This test checks that computeChanges compares the quoted value of the records correctly.
+func TestRoute53NoChange(t *testing.T) {
+	// Existing record set.
+	testTree0 := map[string]recordSet{
+		"n": {ttl: rootTTL, values: []string{
+			`"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`,
+		}},
+		"2xs2367yhaxjfglzhvawlqd4zy.n": {ttl: treeNodeTTL, values: []string{
+			`"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`,
+		}},
+	}
+	// New set.
+	testTree1 := map[string]string{
+		"n":                            "enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA",
+		"2XS2367YHAXJFGLZHVAWLQD4ZY.n": "enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA",
+	}
+
+	var client route53Client
+	changes := client.computeChanges("n", testTree1, testTree0)
+	if len(changes) > 0 {
+		t.Fatalf("wrong changes (got %d, want 0)", len(changes))
+	}
+}
+
 func sp(s string) *string { return &s }
 func ip(i int64) *int64   { return &i }

cmd/devp2p/dnscmd.go

@@ -43,6 +43,7 @@ var (
 			dnsTXTCommand,
 			dnsCloudflareCommand,
 			dnsRoute53Command,
+			dnsRoute53NukeCommand,
 		},
 	}
 	dnsSyncCommand = cli.Command{
@@ -77,7 +78,24 @@ var (
 		Usage:     "Deploy DNS TXT records to Amazon Route53",
 		ArgsUsage: "<tree-directory>",
 		Action:    dnsToRoute53,
-		Flags:     []cli.Flag{route53AccessKeyFlag, route53AccessSecretFlag, route53ZoneIDFlag},
+		Flags: []cli.Flag{
+			route53AccessKeyFlag,
+			route53AccessSecretFlag,
+			route53ZoneIDFlag,
+			route53RegionFlag,
+		},
+	}
+	dnsRoute53NukeCommand = cli.Command{
+		Name:      "nuke-route53",
+		Usage:     "Deletes DNS TXT records of a subdomain on Amazon Route53",
+		ArgsUsage: "<domain>",
+		Action:    dnsNukeRoute53,
+		Flags: []cli.Flag{
+			route53AccessKeyFlag,
+			route53AccessSecretFlag,
+			route53ZoneIDFlag,
+			route53RegionFlag,
+		},
 	}
 )
 
@@ -169,6 +187,9 @@ func dnsSign(ctx *cli.Context) error {
 	return nil
 }
 
+// directoryName returns the directory name of the given path.
+// For example, when dir is "foo/bar", it returns "bar".
+// When dir is ".", and the working directory is "example/foo", it returns "foo".
 func directoryName(dir string) string {
 	abs, err := filepath.Abs(dir)
 	if err != nil {
@@ -177,7 +198,7 @@ func directoryName(dir string) string {
 	return filepath.Base(abs)
 }
 
-// dnsToTXT peforms dnsTXTCommand.
+// dnsToTXT performs dnsTXTCommand.
 func dnsToTXT(ctx *cli.Context) error {
 	if ctx.NArg() < 1 {
 		return fmt.Errorf("need tree definition directory as argument")
@@ -194,9 +215,9 @@ func dnsToTXT(ctx *cli.Context) error {
 	return nil
 }
 
-// dnsToCloudflare peforms dnsCloudflareCommand.
+// dnsToCloudflare performs dnsCloudflareCommand.
 func dnsToCloudflare(ctx *cli.Context) error {
-	if ctx.NArg() < 1 {
+	if ctx.NArg() != 1 {
 		return fmt.Errorf("need tree definition directory as argument")
 	}
 	domain, t, err := loadTreeDefinitionForExport(ctx.Args().Get(0))
@@ -207,9 +228,9 @@ func dnsToCloudflare(ctx *cli.Context) error {
 	return client.deploy(domain, t)
 }
 
-// dnsToRoute53 peforms dnsRoute53Command.
+// dnsToRoute53 performs dnsRoute53Command.
 func dnsToRoute53(ctx *cli.Context) error {
-	if ctx.NArg() < 1 {
+	if ctx.NArg() != 1 {
 		return fmt.Errorf("need tree definition directory as argument")
 	}
 	domain, t, err := loadTreeDefinitionForExport(ctx.Args().Get(0))
@@ -220,6 +241,15 @@ func dnsToRoute53(ctx *cli.Context) error {
 	return client.deploy(domain, t)
 }
 
+// dnsNukeRoute53 performs dnsRoute53NukeCommand.
+func dnsNukeRoute53(ctx *cli.Context) error {
+	if ctx.NArg() != 1 {
+		return fmt.Errorf("need domain name as argument")
+	}
+	client := newRoute53Client(ctx)
+	return client.deleteDomain(ctx.Args().First())
+}
+
 // loadSigningKey loads a private key in Ethereum keystore format.
 func loadSigningKey(keyfile string) *ecdsa.PrivateKey {
 	keyjson, err := ioutil.ReadFile(keyfile)

cmd/devp2p/internal/ethtest/chain.go

@@ -34,6 +34,7 @@ import (
 )
 
 type Chain struct {
+	genesis     core.Genesis
 	blocks      []*types.Block
 	chainConfig *params.ChainConfig
 }
@@ -124,16 +125,34 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) {
 // loadChain takes the given chain.rlp file, and decodes and returns
 // the blocks from the file.
 func loadChain(chainfile string, genesis string) (*Chain, error) {
-	chainConfig, err := ioutil.ReadFile(genesis)
+	gen, err := loadGenesis(genesis)
 	if err != nil {
 		return nil, err
 	}
-	var gen core.Genesis
-	if err := json.Unmarshal(chainConfig, &gen); err != nil {
-		return nil, err
-	}
 	gblock := gen.ToBlock(nil)
 
+	blocks, err := blocksFromFile(chainfile, gblock)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &Chain{genesis: gen, blocks: blocks, chainConfig: gen.Config}
+	return c, nil
+}
+
+func loadGenesis(genesisFile string) (core.Genesis, error) {
+	chainConfig, err := ioutil.ReadFile(genesisFile)
+	if err != nil {
+		return core.Genesis{}, err
+	}
+	var gen core.Genesis
+	if err := json.Unmarshal(chainConfig, &gen); err != nil {
+		return core.Genesis{}, err
+	}
+	return gen, nil
+}
+
+func blocksFromFile(chainfile string, gblock *types.Block) ([]*types.Block, error) {
 	// Load chain.rlp.
 	fh, err := os.Open(chainfile)
 	if err != nil {
@@ -161,7 +180,5 @@ func loadChain(chainfile string, genesis string) (*Chain, error) {
 		}
 		blocks = append(blocks, &b)
 	}
-
-	c := &Chain{blocks: blocks, chainConfig: gen.Config}
-	return c, nil
+	return blocks, nil
 }

cmd/devp2p/internal/ethtest/chain_test.go

@@ -35,7 +35,9 @@ func TestEthProtocolNegotiation(t *testing.T) {
 		expected uint32
 	}{
 		{
-			conn: &Conn{},
+			conn: &Conn{
+				ourHighestProtoVersion: 65,
+			},
 			caps: []p2p.Cap{
 				{Name: "eth", Version: 63},
 				{Name: "eth", Version: 64},
@@ -44,7 +46,42 @@ func TestEthProtocolNegotiation(t *testing.T) {
 			expected: uint32(65),
 		},
 		{
-			conn: &Conn{},
+			conn: &Conn{
+				ourHighestProtoVersion: 65,
+			},
+			caps: []p2p.Cap{
+				{Name: "eth", Version: 63},
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			expected: uint32(65),
+		},
+		{
+			conn: &Conn{
+				ourHighestProtoVersion: 65,
+			},
+			caps: []p2p.Cap{
+				{Name: "eth", Version: 63},
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			expected: uint32(65),
+		},
+		{
+			conn: &Conn{
+				ourHighestProtoVersion: 64,
+			},
+			caps: []p2p.Cap{
+				{Name: "eth", Version: 63},
+				{Name: "eth", Version: 64},
+				{Name: "eth", Version: 65},
+			},
+			expected: 64,
+		},
+		{
+			conn: &Conn{
+				ourHighestProtoVersion: 65,
+			},
 			caps: []p2p.Cap{
 				{Name: "eth", Version: 0},
 				{Name: "eth", Version: 89},
@@ -53,7 +90,20 @@ func TestEthProtocolNegotiation(t *testing.T) {
 			expected: uint32(65),
 		},
 		{
-			conn: &Conn{},
+			conn: &Conn{
+				ourHighestProtoVersion: 64,
+			},
+			caps: []p2p.Cap{
+				{Name: "eth", Version: 63},
+				{Name: "eth", Version: 64},
+				{Name: "wrongProto", Version: 65},
+			},
+			expected: uint32(64),
+		},
+		{
+			conn: &Conn{
+				ourHighestProtoVersion: 65,
+			},
 			caps: []p2p.Cap{
 				{Name: "eth", Version: 63},
 				{Name: "eth", Version: 64},
@@ -66,7 +116,7 @@ func TestEthProtocolNegotiation(t *testing.T) {
 	for i, tt := range tests {
 		t.Run(strconv.Itoa(i), func(t *testing.T) {
 			tt.conn.negotiateEthProtocol(tt.caps)
-			assert.Equal(t, tt.expected, uint32(tt.conn.ethProtocolVersion))
+			assert.Equal(t, tt.expected, uint32(tt.conn.negotiatedProtoVersion))
 		})
 	}
 }

cmd/devp2p/internal/ethtest/eth66_suite.go

@@ -19,6 +19,7 @@ package ethtest
 import (
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/protocols/eth"
@@ -26,11 +27,22 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 )
 
+// Is_66 checks if the node supports the eth66 protocol version,
+// and if not, exists the test suite
+func (s *Suite) Is_66(t *utesting.T) {
+	conn := s.dial66(t)
+	conn.handshake(t)
+	if conn.negotiatedProtoVersion < 66 {
+		t.Fail()
+	}
+}
+
 // TestStatus_66 attempts to connect to the given node and exchange
 // a status message with it on the eth66 protocol, and then check to
 // make sure the chain head is correct.
 func (s *Suite) TestStatus_66(t *utesting.T) {
 	conn := s.dial66(t)
+	defer conn.Close()
 	// get protoHandshake
 	conn.handshake(t)
 	// get status
@@ -50,6 +62,7 @@ func (s *Suite) TestStatus_66(t *utesting.T) {
 // an eth66 `GetBlockHeaders` request and that the response is accurate.
 func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) {
 	conn := s.setupConnection66(t)
+	defer conn.Close()
 	// get block headers
 	req := &eth.GetBlockHeadersPacket66{
 		RequestId: 3,
@@ -63,9 +76,14 @@ func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) {
 		},
 	}
 	// write message
-	headers := s.getBlockHeaders66(t, conn, req, req.RequestId)
+	headers, err := s.getBlockHeaders66(conn, req, req.RequestId)
+	if err != nil {
+		t.Fatalf("could not get block headers: %v", err)
+	}
 	// check for correct headers
-	headersMatch(t, s.chain, headers)
+	if !headersMatch(t, s.chain, headers) {
+		t.Fatal("received wrong header(s)")
+	}
 }
 
 // TestSimultaneousRequests_66 sends two simultaneous `GetBlockHeader` requests
@@ -73,7 +91,8 @@ func (s *Suite) TestGetBlockHeaders_66(t *utesting.T) {
 // headers per request.
 func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) {
 	// create two connections
-	conn1, conn2 := s.setupConnection66(t), s.setupConnection66(t)
+	conn := s.setupConnection66(t)
+	defer conn.Close()
 	// create two requests
 	req1 := &eth.GetBlockHeadersPacket66{
 		RequestId: 111,
@@ -97,33 +116,36 @@ func (s *Suite) TestSimultaneousRequests_66(t *utesting.T) {
 			Reverse: false,
 		},
 	}
-	// wait for headers for first request
-	headerChan := make(chan BlockHeaders, 1)
-	go func(headers chan BlockHeaders) {
-		headers <- s.getBlockHeaders66(t, conn1, req1, req1.RequestId)
-	}(headerChan)
-	// check headers of second request
-	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn2, req2, req2.RequestId))
-	// check headers of first request
-	headersMatch(t, s.chain, <-headerChan)
+	// write first request
+	if err := conn.write66(req1, GetBlockHeaders{}.Code()); err != nil {
+		t.Fatalf("failed to write to connection: %v", err)
+	}
+	// write second request
+	if err := conn.write66(req2, GetBlockHeaders{}.Code()); err != nil {
+		t.Fatalf("failed to write to connection: %v", err)
+	}
+	// wait for responses
+	headers1, err := s.waitForBlockHeadersResponse66(conn, req1.RequestId)
+	if err != nil {
+		t.Fatalf("error while waiting for block headers: %v", err)
+	}
+	headers2, err := s.waitForBlockHeadersResponse66(conn, req2.RequestId)
+	if err != nil {
+		t.Fatalf("error while waiting for block headers: %v", err)
+	}
+	// check headers of both responses
+	if !headersMatch(t, s.chain, headers1) {
+		t.Fatalf("wrong header(s) in response to req1: got %v", headers1)
+	}
+	if !headersMatch(t, s.chain, headers2) {
+		t.Fatalf("wrong header(s) in response to req2: got %v", headers2)
+	}
 }
 
 // TestBroadcast_66 tests whether a block announcement is correctly
 // propagated to the given node's peer(s) on the eth66 protocol.
 func (s *Suite) TestBroadcast_66(t *utesting.T) {
-	sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t)
-	nextBlock := len(s.chain.blocks)
-	blockAnnouncement := &NewBlock{
-		Block: s.fullChain.blocks[nextBlock],
-		TD:    s.fullChain.TD(nextBlock + 1),
-	}
-	s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement)
-	// update test suite chain
-	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
-	// wait for client to update its chain
-	if err := receiveConn.waitForBlock66(s.chain.Head()); err != nil {
-		t.Fatal(err)
-	}
+	s.sendNextBlock66(t)
 }
 
 // TestGetBlockBodies_66 tests whether the given node can respond to
@@ -131,6 +153,7 @@ func (s *Suite) TestBroadcast_66(t *utesting.T) {
 // the eth66 protocol.
 func (s *Suite) TestGetBlockBodies_66(t *utesting.T) {
 	conn := s.setupConnection66(t)
+	defer conn.Close()
 	// create block bodies request
 	id := uint64(55)
 	req := &eth.GetBlockBodiesPacket66{
@@ -185,29 +208,31 @@ func (s *Suite) TestLargeAnnounce_66(t *utesting.T) {
 			t.Fatalf("could not write to connection: %v", err)
 		}
 		// Invalid announcement, check that peer disconnected
-		switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) {
+		switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) {
 		case *Disconnect:
 		case *Error:
 			break
 		default:
 			t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg))
 		}
+		sendConn.Close()
 	}
 	// Test the last block as a valid block
-	sendConn := s.setupConnection66(t)
-	receiveConn := s.setupConnection66(t)
-	s.testAnnounce66(t, sendConn, receiveConn, blocks[3])
-	// update test suite chain
-	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
-	// wait for client to update its chain
-	if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil {
-		t.Fatal(err)
+	s.sendNextBlock66(t)
 }
+
+func (s *Suite) TestOldAnnounce_66(t *utesting.T) {
+	sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t)
+	defer sendConn.Close()
+	defer recvConn.Close()
+
+	s.oldAnnounce(t, sendConn, recvConn)
 }
 
 // TestMaliciousHandshake_66 tries to send malicious data during the handshake.
 func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) {
 	conn := s.dial66(t)
+	defer conn.Close()
 	// write hello to client
 	pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:]
 	handshakes := []*Hello{
@@ -281,6 +306,7 @@ func (s *Suite) TestMaliciousHandshake_66(t *utesting.T) {
 // TestMaliciousStatus_66 sends a status package with a large total difficulty.
 func (s *Suite) TestMaliciousStatus_66(t *utesting.T) {
 	conn := s.dial66(t)
+	defer conn.Close()
 	// get protoHandshake
 	conn.handshake(t)
 	status := &Status{
@@ -320,23 +346,37 @@ func (s *Suite) TestTransaction_66(t *utesting.T) {
 }
 
 func (s *Suite) TestMaliciousTx_66(t *utesting.T) {
-	tests := []*types.Transaction{
+	badTxs := []*types.Transaction{
 		getOldTxFromChain(t, s),
 		invalidNonceTx(t, s),
 		hugeAmount(t, s),
 		hugeGasPrice(t, s),
 		hugeData(t, s),
 	}
-	for i, tx := range tests {
+	sendConn := s.setupConnection66(t)
+	defer sendConn.Close()
+	// set up receiving connection before sending txs to make sure
+	// no announcements are missed
+	recvConn := s.setupConnection66(t)
+	defer recvConn.Close()
+
+	for i, tx := range badTxs {
 		t.Logf("Testing malicious tx propagation: %v\n", i)
-		sendFailingTx66(t, s, tx)
+		if err := sendConn.Write(&Transactions{tx}); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
 		}
+
+	}
+	// check to make sure bad txs aren't propagated
+	waitForTxPropagation(t, s, badTxs, recvConn)
 }
 
 // TestZeroRequestID_66 checks that a request ID of zero is still handled
 // by the node.
 func (s *Suite) TestZeroRequestID_66(t *utesting.T) {
 	conn := s.setupConnection66(t)
+	defer conn.Close()
+
 	req := &eth.GetBlockHeadersPacket66{
 		RequestId: 0,
 		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
@@ -346,25 +386,31 @@ func (s *Suite) TestZeroRequestID_66(t *utesting.T) {
 			Amount: 2,
 		},
 	}
-	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req, req.RequestId))
+	headers, err := s.getBlockHeaders66(conn, req, req.RequestId)
+	if err != nil {
+		t.Fatalf("could not get block headers: %v", err)
+	}
+	if !headersMatch(t, s.chain, headers) {
+		t.Fatal("received wrong header(s)")
+	}
 }
 
 // TestSameRequestID_66 sends two requests with the same request ID
 // concurrently to a single node.
 func (s *Suite) TestSameRequestID_66(t *utesting.T) {
 	conn := s.setupConnection66(t)
-	// create two separate requests with same ID
+	// create two requests with the same request ID
 	reqID := uint64(1234)
-	req1 := &eth.GetBlockHeadersPacket66{
+	request1 := &eth.GetBlockHeadersPacket66{
 		RequestId: reqID,
 		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
 			Origin: eth.HashOrNumber{
-				Number: 0,
+				Number: 1,
 			},
 			Amount: 2,
 		},
 	}
-	req2 := &eth.GetBlockHeadersPacket66{
+	request2 := &eth.GetBlockHeadersPacket66{
 		RequestId: reqID,
 		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
 			Origin: eth.HashOrNumber{
@@ -373,10 +419,103 @@ func (s *Suite) TestSameRequestID_66(t *utesting.T) {
 			Amount: 2,
 		},
 	}
-	// send requests concurrently
-	go func() {
-		headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req2, reqID))
-	}()
-	// check response from first request
-	headersMatch(t, s.chain, s.getBlockHeaders66(t, conn, req1, reqID))
+	// write the first request
+	err := conn.write66(request1, GetBlockHeaders{}.Code())
+	if err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	// perform second request
+	headers2, err := s.getBlockHeaders66(conn, request2, reqID)
+	if err != nil {
+		t.Fatalf("could not get block headers: %v", err)
+		return
+	}
+	// wait for response to first request
+	headers1, err := s.waitForBlockHeadersResponse66(conn, reqID)
+	if err != nil {
+		t.Fatalf("could not get BlockHeaders response: %v", err)
+	}
+	// check if headers match
+	if !headersMatch(t, s.chain, headers1) || !headersMatch(t, s.chain, headers2) {
+		t.Fatal("received wrong header(s)")
+	}
+}
+
+// TestLargeTxRequest_66 tests whether a node can fulfill a large GetPooledTransactions
+// request.
+func (s *Suite) TestLargeTxRequest_66(t *utesting.T) {
+	// send the next block to ensure the node is no longer syncing and is able to accept
+	// txs
+	s.sendNextBlock66(t)
+	// send 2000 transactions to the node
+	hashMap, txs := generateTxs(t, s, 2000)
+	sendConn := s.setupConnection66(t)
+	defer sendConn.Close()
+
+	sendMultipleSuccessfulTxs(t, s, sendConn, txs)
+	// set up connection to receive to ensure node is peered with the receiving connection
+	// before tx request is sent
+	recvConn := s.setupConnection66(t)
+	defer recvConn.Close()
+	// create and send pooled tx request
+	hashes := make([]common.Hash, 0)
+	for _, hash := range hashMap {
+		hashes = append(hashes, hash)
+	}
+	getTxReq := &eth.GetPooledTransactionsPacket66{
+		RequestId:                   1234,
+		GetPooledTransactionsPacket: hashes,
+	}
+	if err := recvConn.write66(getTxReq, GetPooledTransactions{}.Code()); err != nil {
+		t.Fatalf("could not write to conn: %v", err)
+	}
+	// check that all received transactions match those that were sent to node
+	switch msg := recvConn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) {
+	case PooledTransactions:
+		for _, gotTx := range msg {
+			if _, exists := hashMap[gotTx.Hash()]; !exists {
+				t.Fatalf("unexpected tx received: %v", gotTx.Hash())
+			}
+		}
+	default:
+		t.Fatalf("unexpected %s", pretty.Sdump(msg))
+	}
+}
+
+// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions
+// request upon receiving a NewPooledTransactionHashes announcement.
+func (s *Suite) TestNewPooledTxs_66(t *utesting.T) {
+	// send the next block to ensure the node is no longer syncing and is able to accept
+	// txs
+	s.sendNextBlock66(t)
+	// generate 50 txs
+	hashMap, _ := generateTxs(t, s, 50)
+	// create new pooled tx hashes announcement
+	hashes := make([]common.Hash, 0)
+	for _, hash := range hashMap {
+		hashes = append(hashes, hash)
+	}
+	announce := NewPooledTransactionHashes(hashes)
+	// send announcement
+	conn := s.setupConnection66(t)
+	defer conn.Close()
+	if err := conn.Write(announce); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	// wait for GetPooledTxs request
+	for {
+		_, msg := conn.readAndServe66(s.chain, timeout)
+		switch msg := msg.(type) {
+		case GetPooledTransactions:
+			if len(msg) != len(hashes) {
+				t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg))
+			}
+			return
+		case *NewPooledTransactionHashes, *NewBlock, *NewBlockHashes:
+			// ignore propagated txs and blocks from old tests
+			continue
+		default:
+			t.Fatalf("unexpected %s", pretty.Sdump(msg))
+		}
+	}
 }

cmd/devp2p/internal/ethtest/eth66_suiteHelpers.go

@@ -18,6 +18,7 @@ package ethtest
 
 import (
 	"fmt"
+	"reflect"
 	"time"
 
 	"github.com/ethereum/go-ethereum/core/types"
@@ -46,6 +47,7 @@ func (s *Suite) dial66(t *utesting.T) *Conn {
 		t.Fatalf("could not dial: %v", err)
 	}
 	conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66})
+	conn.ourHighestProtoVersion = 66
 	return conn
 }
 
@@ -110,6 +112,18 @@ func (c *Conn) read66() (uint64, Message) {
 		msg = new(Transactions)
 	case (NewPooledTransactionHashes{}).Code():
 		msg = new(NewPooledTransactionHashes)
+	case (GetPooledTransactions{}.Code()):
+		ethMsg := new(eth.GetPooledTransactionsPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket)
+	case (PooledTransactions{}.Code()):
+		ethMsg := new(eth.PooledTransactionsPacket66)
+		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
+			return 0, errorf("could not rlp decode message: %v", err)
+		}
+		return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket)
 	default:
 		msg = errorf("invalid message code: %d", code)
 	}
@@ -123,13 +137,21 @@ func (c *Conn) read66() (uint64, Message) {
 	return 0, errorf("invalid message: %s", string(rawData))
 }
 
+func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message {
+	for {
+		id, msg := c.readAndServe66(chain, timeout)
+		if id == requestID {
+			return msg
+		}
+	}
+}
+
 // ReadAndServe serves GetBlockHeaders requests while waiting
 // on another message from the node.
 func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) {
 	start := time.Now()
 	for time.Since(start) < timeout {
-		timeout := time.Now().Add(10 * time.Second)
-		c.SetReadDeadline(timeout)
+		c.SetReadDeadline(time.Now().Add(10 * time.Second))
 
 		reqID, msg := c.read66()
 
@@ -141,8 +163,11 @@ func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Mess
 			if err != nil {
 				return 0, errorf("could not get headers for inbound header request: %v", err)
 			}
-
-			if err := c.Write(headers); err != nil {
+			resp := &eth.BlockHeadersPacket66{
+				RequestId:          reqID,
+				BlockHeadersPacket: eth.BlockHeadersPacket(headers),
+			}
+			if err := c.write66(resp, BlockHeaders{}.Code()); err != nil {
 				return 0, errorf("could not write to connection: %v", err)
 			}
 		default:
@@ -169,7 +194,7 @@ func (s *Suite) testAnnounce66(t *utesting.T, sendConn, receiveConn *Conn, block
 }
 
 func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) {
-	timeout := 20 * time.Second
+	for {
 		_, msg := conn.readAndServe66(s.chain, timeout)
 		switch msg := msg.(type) {
 		case *NewBlock:
@@ -182,24 +207,33 @@ func (s *Suite) waitAnnounce66(t *utesting.T, conn *Conn, blockAnnouncement *New
 				blockAnnouncement.TD, msg.TD,
 				"wrong TD in announcement",
 			)
+			return
 		case *NewBlockHashes:
 			blockHashes := *msg
 			t.Logf("received NewBlockHashes message: %s", pretty.Sdump(blockHashes))
 			assert.Equal(t, blockAnnouncement.Block.Hash(), blockHashes[0].Hash,
 				"wrong block hash in announcement",
 			)
+			return
+		case *NewPooledTransactionHashes:
+			// ignore old txs being propagated
+			continue
 		default:
 			t.Fatalf("unexpected: %s", pretty.Sdump(msg))
 		}
 	}
+}
 
 // waitForBlock66 waits for confirmation from the client that it has
 // imported the given block.
 func (c *Conn) waitForBlock66(block *types.Block) error {
 	defer c.SetReadDeadline(time.Time{})
 
-	timeout := time.Now().Add(20 * time.Second)
-	c.SetReadDeadline(timeout)
+	c.SetReadDeadline(time.Now().Add(20 * time.Second))
+	// note: if the node has not yet imported the block, it will respond
+	// to the GetBlockHeaders request with an empty BlockHeaders response,
+	// so the GetBlockHeaders request must be sent again until the BlockHeaders
+	// response contains the desired header.
 	for {
 		req := eth.GetBlockHeadersPacket66{
 			RequestId: 54,
@@ -222,10 +256,15 @@ func (c *Conn) waitForBlock66(block *types.Block) error {
 			if reqID != req.RequestId {
 				return fmt.Errorf("request ID mismatch: wanted %d, got %d", req.RequestId, reqID)
 			}
-			if len(msg) > 0 {
+			for _, header := range msg {
+				if header.Number.Uint64() == block.NumberU64() {
 					return nil
 				}
+			}
 			time.Sleep(100 * time.Millisecond)
+		case *NewPooledTransactionHashes:
+			// ignore old announcements
+			continue
 		default:
 			return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))
 		}
@@ -234,37 +273,61 @@ func (c *Conn) waitForBlock66(block *types.Block) error {
 
 func sendSuccessfulTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
 	sendConn := s.setupConnection66(t)
+	defer sendConn.Close()
 	sendSuccessfulTxWithConn(t, s, tx, sendConn)
 }
 
-func sendFailingTx66(t *utesting.T, s *Suite, tx *types.Transaction) {
-	sendConn, recvConn := s.setupConnection66(t), s.setupConnection66(t)
-	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
-}
-
-func (s *Suite) getBlockHeaders66(t *utesting.T, conn *Conn, req eth.Packet, expectedID uint64) BlockHeaders {
-	if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil {
-		t.Fatalf("could not write to connection: %v", err)
-	}
-	// check block headers response
+// waitForBlockHeadersResponse66 waits for a BlockHeaders message with the given expected request ID
+func (s *Suite) waitForBlockHeadersResponse66(conn *Conn, expectedID uint64) (BlockHeaders, error) {
 	reqID, msg := conn.readAndServe66(s.chain, timeout)
-
 	switch msg := msg.(type) {
 	case BlockHeaders:
 		if reqID != expectedID {
-			t.Fatalf("request ID mismatch: wanted %d, got %d", expectedID, reqID)
+			return nil, fmt.Errorf("request ID mismatch: wanted %d, got %d", expectedID, reqID)
 		}
-		return msg
+		return msg, nil
 	default:
-		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
-		return nil
+		return nil, fmt.Errorf("unexpected: %s", pretty.Sdump(msg))
 	}
 }
 
-func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) {
+func (s *Suite) getBlockHeaders66(conn *Conn, req eth.Packet, expectedID uint64) (BlockHeaders, error) {
+	if err := conn.write66(req, GetBlockHeaders{}.Code()); err != nil {
+		return nil, fmt.Errorf("could not write to connection: %v", err)
+	}
+	return s.waitForBlockHeadersResponse66(conn, expectedID)
+}
+
+func headersMatch(t *utesting.T, chain *Chain, headers BlockHeaders) bool {
+	mismatched := 0
 	for _, header := range headers {
 		num := header.Number.Uint64()
 		t.Logf("received header (%d): %s", num, pretty.Sdump(header.Hash()))
-		assert.Equal(t, chain.blocks[int(num)].Header(), header)
+		if !reflect.DeepEqual(chain.blocks[int(num)].Header(), header) {
+			mismatched += 1
+			t.Logf("received wrong header: %v", pretty.Sdump(header))
 		}
 	}
+	return mismatched == 0
+}
+
+func (s *Suite) sendNextBlock66(t *utesting.T) {
+	sendConn, receiveConn := s.setupConnection66(t), s.setupConnection66(t)
+	defer sendConn.Close()
+	defer receiveConn.Close()
+
+	// create new block announcement
+	nextBlock := len(s.chain.blocks)
+	blockAnnouncement := &NewBlock{
+		Block: s.fullChain.blocks[nextBlock],
+		TD:    s.fullChain.TD(nextBlock + 1),
+	}
+	// send announcement and wait for node to request the header
+	s.testAnnounce66(t, sendConn, receiveConn, blockAnnouncement)
+	// wait for client to update its chain
+	if err := receiveConn.waitForBlock66(s.fullChain.blocks[nextBlock]); err != nil {
+		t.Fatal(err)
+	}
+	// update test suite chain
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
+}

cmd/devp2p/internal/ethtest/large.go

@@ -70,7 +70,7 @@ func largeHeader() *types.Header {
 		GasUsed:     0,
 		Coinbase:    common.Address{},
 		GasLimit:    0,
-		UncleHash:   randHash(),
+		UncleHash:   types.EmptyUncleHash,
 		Time:        1337,
 		ParentHash:  randHash(),
 		Root:        randHash(),

cmd/devp2p/internal/ethtest/suite.go

@@ -19,6 +19,7 @@ package ethtest
 import (
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
@@ -65,35 +66,73 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e
 	}, nil
 }
 
-func (s *Suite) EthTests() []utesting.Test {
+func (s *Suite) AllEthTests() []utesting.Test {
 	return []utesting.Test{
 		// status
-		{Name: "Status", Fn: s.TestStatus},
-		{Name: "Status_66", Fn: s.TestStatus_66},
+		{Name: "TestStatus", Fn: s.TestStatus},
+		{Name: "TestStatus_66", Fn: s.TestStatus_66},
 		// get block headers
-		{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
-		{Name: "GetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66},
+		{Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders},
+		{Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66},
 		{Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66},
 		{Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66},
 		{Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66},
 		// get block bodies
-		{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
-		{Name: "GetBlockBodies_66", Fn: s.TestGetBlockBodies_66},
+		{Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies},
+		{Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66},
 		// broadcast
-		{Name: "Broadcast", Fn: s.TestBroadcast},
-		{Name: "Broadcast_66", Fn: s.TestBroadcast_66},
+		{Name: "TestBroadcast", Fn: s.TestBroadcast},
+		{Name: "TestBroadcast_66", Fn: s.TestBroadcast_66},
 		{Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce},
 		{Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66},
+		{Name: "TestOldAnnounce", Fn: s.TestOldAnnounce},
+		{Name: "TestOldAnnounce_66", Fn: s.TestOldAnnounce_66},
 		// malicious handshakes + status
 		{Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake},
 		{Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus},
 		{Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66},
-		{Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus},
+		{Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66},
 		// test transactions
-		{Name: "TestTransactions", Fn: s.TestTransaction},
-		{Name: "TestTransactions_66", Fn: s.TestTransaction_66},
-		{Name: "TestMaliciousTransactions", Fn: s.TestMaliciousTx},
-		{Name: "TestMaliciousTransactions_66", Fn: s.TestMaliciousTx_66},
+		{Name: "TestTransaction", Fn: s.TestTransaction},
+		{Name: "TestTransaction_66", Fn: s.TestTransaction_66},
+		{Name: "TestMaliciousTx", Fn: s.TestMaliciousTx},
+		{Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66},
+		{Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66},
+		{Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66},
+	}
+}
+
+func (s *Suite) EthTests() []utesting.Test {
+	return []utesting.Test{
+		{Name: "TestStatus", Fn: s.TestStatus},
+		{Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders},
+		{Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies},
+		{Name: "TestBroadcast", Fn: s.TestBroadcast},
+		{Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce},
+		{Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake},
+		{Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus},
+		{Name: "TestTransaction", Fn: s.TestTransaction},
+		{Name: "TestMaliciousTx", Fn: s.TestMaliciousTx},
+	}
+}
+
+func (s *Suite) Eth66Tests() []utesting.Test {
+	return []utesting.Test{
+		// only proceed with eth66 test suite if node supports eth 66 protocol
+		{Name: "TestStatus_66", Fn: s.TestStatus_66},
+		{Name: "TestGetBlockHeaders_66", Fn: s.TestGetBlockHeaders_66},
+		{Name: "TestSimultaneousRequests_66", Fn: s.TestSimultaneousRequests_66},
+		{Name: "TestSameRequestID_66", Fn: s.TestSameRequestID_66},
+		{Name: "TestZeroRequestID_66", Fn: s.TestZeroRequestID_66},
+		{Name: "TestGetBlockBodies_66", Fn: s.TestGetBlockBodies_66},
+		{Name: "TestBroadcast_66", Fn: s.TestBroadcast_66},
+		{Name: "TestLargeAnnounce_66", Fn: s.TestLargeAnnounce_66},
+		{Name: "TestMaliciousHandshake_66", Fn: s.TestMaliciousHandshake_66},
+		{Name: "TestMaliciousStatus_66", Fn: s.TestMaliciousStatus_66},
+		{Name: "TestTransaction_66", Fn: s.TestTransaction_66},
+		{Name: "TestMaliciousTx_66", Fn: s.TestMaliciousTx_66},
+		{Name: "TestLargeTxRequest_66", Fn: s.TestLargeTxRequest_66},
+		{Name: "TestNewPooledTxs_66", Fn: s.TestNewPooledTxs_66},
 	}
 }
 
@@ -105,6 +144,7 @@ func (s *Suite) TestStatus(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("could not dial: %v", err)
 	}
+	defer conn.Close()
 	// get protoHandshake
 	conn.handshake(t)
 	// get status
@@ -122,10 +162,11 @@ func (s *Suite) TestMaliciousStatus(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("could not dial: %v", err)
 	}
+	defer conn.Close()
 	// get protoHandshake
 	conn.handshake(t)
 	status := &Status{
-		ProtocolVersion: uint32(conn.ethProtocolVersion),
+		ProtocolVersion: uint32(conn.negotiatedProtoVersion),
 		NetworkID:       s.chain.chainConfig.ChainID.Uint64(),
 		TD:              largeNumber(2),
 		Head:            s.chain.blocks[s.chain.Len()-1].Hash(),
@@ -156,6 +197,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("could not dial: %v", err)
 	}
+	defer conn.Close()
 
 	conn.handshake(t)
 	conn.statusExchange(t, s.chain, nil)
@@ -194,6 +236,7 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("could not dial: %v", err)
 	}
+	defer conn.Close()
 
 	conn.handshake(t)
 	conn.statusExchange(t, s.chain, nil)
@@ -217,19 +260,28 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 // TestBroadcast tests whether a block announcement is correctly
 // propagated to the given node's peer(s).
 func (s *Suite) TestBroadcast(t *utesting.T) {
+	s.sendNextBlock(t)
+}
+
+func (s *Suite) sendNextBlock(t *utesting.T) {
 	sendConn, receiveConn := s.setupConnection(t), s.setupConnection(t)
+	defer sendConn.Close()
+	defer receiveConn.Close()
+
+	// create new block announcement
 	nextBlock := len(s.chain.blocks)
 	blockAnnouncement := &NewBlock{
 		Block: s.fullChain.blocks[nextBlock],
 		TD:    s.fullChain.TD(nextBlock + 1),
 	}
+	// send announcement and wait for node to request the header
 	s.testAnnounce(t, sendConn, receiveConn, blockAnnouncement)
-	// update test suite chain
-	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
 	// wait for client to update its chain
-	if err := receiveConn.waitForBlock(s.chain.Head()); err != nil {
+	if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil {
 		t.Fatal(err)
 	}
+	// update test suite chain
+	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
 }
 
 // TestMaliciousHandshake tries to send malicious data during the handshake.
@@ -238,6 +290,7 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("could not dial: %v", err)
 	}
+	defer conn.Close()
 	// write hello to client
 	pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:]
 	handshakes := []*Hello{
@@ -337,23 +390,58 @@ func (s *Suite) TestLargeAnnounce(t *utesting.T) {
 			t.Fatalf("could not write to connection: %v", err)
 		}
 		// Invalid announcement, check that peer disconnected
-		switch msg := sendConn.ReadAndServe(s.chain, timeout).(type) {
+		switch msg := sendConn.ReadAndServe(s.chain, time.Second*8).(type) {
 		case *Disconnect:
 		case *Error:
 			break
 		default:
 			t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg))
 		}
+		sendConn.Close()
 	}
 	// Test the last block as a valid block
-	sendConn := s.setupConnection(t)
-	receiveConn := s.setupConnection(t)
-	s.testAnnounce(t, sendConn, receiveConn, blocks[3])
-	// update test suite chain
-	s.chain.blocks = append(s.chain.blocks, s.fullChain.blocks[nextBlock])
-	// wait for client to update its chain
-	if err := receiveConn.waitForBlock(s.fullChain.blocks[nextBlock]); err != nil {
-		t.Fatal(err)
+	s.sendNextBlock(t)
+}
+
+func (s *Suite) TestOldAnnounce(t *utesting.T) {
+	sendConn, recvConn := s.setupConnection(t), s.setupConnection(t)
+	defer sendConn.Close()
+	defer recvConn.Close()
+
+	s.oldAnnounce(t, sendConn, recvConn)
+}
+
+func (s *Suite) oldAnnounce(t *utesting.T, sendConn, receiveConn *Conn) {
+	oldBlockAnnounce := &NewBlock{
+		Block: s.chain.blocks[len(s.chain.blocks)/2],
+		TD:    s.chain.blocks[len(s.chain.blocks)/2].Difficulty(),
+	}
+
+	if err := sendConn.Write(oldBlockAnnounce); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+
+	switch msg := receiveConn.ReadAndServe(s.chain, time.Second*8).(type) {
+	case *NewBlock:
+		block := *msg
+		if block.Block.Hash() == oldBlockAnnounce.Block.Hash() {
+			t.Fatalf("unexpected: block propagated: %s", pretty.Sdump(msg))
+		}
+	case *NewBlockHashes:
+		hashes := *msg
+		for _, hash := range hashes {
+			if hash.Hash == oldBlockAnnounce.Block.Hash() {
+				t.Fatalf("unexpected: block announced: %s", pretty.Sdump(msg))
+			}
+		}
+	case *Error:
+		errMsg := *msg
+		// check to make sure error is timeout (propagation didn't come through == test successful)
+		if !strings.Contains(errMsg.String(), "timeout") {
+			t.Fatalf("unexpected error: %v", pretty.Sdump(msg))
+		}
+	default:
+		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
 	}
 }
 
@@ -366,7 +454,6 @@ func (s *Suite) testAnnounce(t *utesting.T, sendConn, receiveConn *Conn, blockAn
 }
 
 func (s *Suite) waitAnnounce(t *utesting.T, conn *Conn, blockAnnouncement *NewBlock) {
-	timeout := 20 * time.Second
 	switch msg := conn.ReadAndServe(s.chain, timeout).(type) {
 	case *NewBlock:
 		t.Logf("received NewBlock message: %s", pretty.Sdump(msg.Block))
@@ -421,6 +508,7 @@ func (s *Suite) dial() (*Conn, error) {
 		{Name: "eth", Version: 64},
 		{Name: "eth", Version: 65},
 	}
+	conn.ourHighestProtoVersion = 65
 	return &conn, nil
 }
 
@@ -436,15 +524,27 @@ func (s *Suite) TestTransaction(t *utesting.T) {
 }
 
 func (s *Suite) TestMaliciousTx(t *utesting.T) {
-	tests := []*types.Transaction{
+	badTxs := []*types.Transaction{
 		getOldTxFromChain(t, s),
 		invalidNonceTx(t, s),
 		hugeAmount(t, s),
 		hugeGasPrice(t, s),
 		hugeData(t, s),
 	}
-	for i, tx := range tests {
+	sendConn := s.setupConnection(t)
+	defer sendConn.Close()
+	// set up receiving connection before sending txs to make sure
+	// no announcements are missed
+	recvConn := s.setupConnection(t)
+	defer recvConn.Close()
+
+	for i, tx := range badTxs {
 		t.Logf("Testing malicious tx propagation: %v\n", i)
-		sendFailingTx(t, s, tx)
+		if err := sendConn.Write(&Transactions{tx}); err != nil {
+			t.Fatalf("could not write to connection: %v", err)
 		}
+
+	}
+	// check to make sure bad txs aren't propagated
+	waitForTxPropagation(t, s, badTxs, recvConn)
 }

cmd/devp2p/internal/ethtest/suite_test.go

@@ -0,0 +1,107 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethtest
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+)
+
+var (
+	genesisFile   = "./testdata/genesis.json"
+	halfchainFile = "./testdata/halfchain.rlp"
+	fullchainFile = "./testdata/chain.rlp"
+)
+
+func TestEthSuite(t *testing.T) {
+	geth, err := runGeth()
+	if err != nil {
+		t.Fatalf("could not run geth: %v", err)
+	}
+	defer geth.Close()
+
+	suite, err := NewSuite(geth.Server().Self(), fullchainFile, genesisFile)
+	if err != nil {
+		t.Fatalf("could not create new test suite: %v", err)
+	}
+	for _, test := range suite.AllEthTests() {
+		t.Run(test.Name, func(t *testing.T) {
+			result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout)
+			if result[0].Failed {
+				t.Fatal()
+			}
+		})
+	}
+}
+
+// runGeth creates and starts a geth node
+func runGeth() (*node.Node, error) {
+	stack, err := node.New(&node.Config{
+		P2P: p2p.Config{
+			ListenAddr:  "127.0.0.1:0",
+			NoDiscovery: true,
+			MaxPeers:    10, // in case a test requires multiple connections, can be changed in the future
+			NoDial:      true,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = setupGeth(stack)
+	if err != nil {
+		stack.Close()
+		return nil, err
+	}
+	if err = stack.Start(); err != nil {
+		stack.Close()
+		return nil, err
+	}
+	return stack, nil
+}
+
+func setupGeth(stack *node.Node) error {
+	chain, err := loadChain(halfchainFile, genesisFile)
+	if err != nil {
+		return err
+	}
+
+	backend, err := eth.New(stack, &ethconfig.Config{
+		Genesis:                 &chain.genesis,
+		NetworkId:               chain.genesis.Config.ChainID.Uint64(), // 19763
+		DatabaseCache:           10,
+		TrieCleanCache:          10,
+		TrieCleanCacheJournal:   "",
+		TrieCleanCacheRejournal: 60 * time.Minute,
+		TrieDirtyCache:          16,
+		TrieTimeout:             60 * time.Minute,
+		SnapshotCache:           10,
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = backend.BlockChain().InsertChain(chain.blocks[1:])
+	return err
+}

cmd/devp2p/internal/ethtest/transaction.go

@@ -17,12 +17,15 @@
 package ethtest
 
 import (
+	"math/big"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 //var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7")
@@ -30,6 +33,7 @@ var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c666
 
 func sendSuccessfulTx(t *utesting.T, s *Suite, tx *types.Transaction) {
 	sendConn := s.setupConnection(t)
+	defer sendConn.Close()
 	sendSuccessfulTxWithConn(t, s, tx, sendConn)
 }
 
@@ -39,7 +43,9 @@ func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, se
 	if err := sendConn.Write(&Transactions{tx}); err != nil {
 		t.Fatal(err)
 	}
-	time.Sleep(100 * time.Millisecond)
+	// update last nonce seen
+	nonce = tx.Nonce()
+
 	recvConn := s.setupConnection(t)
 	// Wait for the transaction announcement
 	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
@@ -65,29 +71,84 @@ func sendSuccessfulTxWithConn(t *utesting.T, s *Suite, tx *types.Transaction, se
 	}
 }
 
-func sendFailingTx(t *utesting.T, s *Suite, tx *types.Transaction) {
-	sendConn, recvConn := s.setupConnection(t), s.setupConnection(t)
-	sendFailingTxWithConns(t, s, tx, sendConn, recvConn)
-}
+var nonce = uint64(99)
 
-func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, sendConn, recvConn *Conn) {
-	// Wait for a transaction announcement
-	switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
-	case *NewPooledTransactionHashes:
-		break
-	default:
-		t.Logf("unexpected message, logging: %v", pretty.Sdump(msg))
-	}
-	// Send the transaction
-	if err := sendConn.Write(&Transactions{tx}); err != nil {
+func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, sendConn *Conn, txs []*types.Transaction) {
+	txMsg := Transactions(txs)
+	t.Logf("sending %d txs\n", len(txs))
+
+	recvConn := s.setupConnection(t)
+	defer recvConn.Close()
+
+	// Send the transactions
+	if err := sendConn.Write(&txMsg); err != nil {
 		t.Fatal(err)
 	}
-	// Wait for another transaction announcement
+	// update nonce
+	nonce = txs[len(txs)-1].Nonce()
+	// Wait for the transaction announcement(s) and make sure all sent txs are being propagated
+	recvHashes := make([]common.Hash, 0)
+	// all txs should be announced within 3 announcements
+	for i := 0; i < 3; i++ {
 		switch msg := recvConn.ReadAndServe(s.chain, timeout).(type) {
 		case *Transactions:
-		t.Fatalf("Received unexpected transaction announcement: %v", msg)
+			for _, tx := range *msg {
+				recvHashes = append(recvHashes, tx.Hash())
+			}
 		case *NewPooledTransactionHashes:
-		t.Fatalf("Received unexpected pooledTx announcement: %v", msg)
+			recvHashes = append(recvHashes, *msg...)
+		default:
+			if !strings.Contains(pretty.Sdump(msg), "i/o timeout") {
+				t.Fatalf("unexpected message while waiting to receive txs: %s", pretty.Sdump(msg))
+			}
+		}
+		// break once all 2000 txs have been received
+		if len(recvHashes) == 2000 {
+			break
+		}
+		if len(recvHashes) > 0 {
+			_, missingTxs := compareReceivedTxs(recvHashes, txs)
+			if len(missingTxs) > 0 {
+				continue
+			} else {
+				t.Logf("successfully received all %d txs", len(txs))
+				return
+			}
+		}
+	}
+	_, missingTxs := compareReceivedTxs(recvHashes, txs)
+	if len(missingTxs) > 0 {
+		for _, missing := range missingTxs {
+			t.Logf("missing tx: %v", missing.Hash())
+		}
+		t.Fatalf("missing %d txs", len(missingTxs))
+	}
+}
+
+func waitForTxPropagation(t *utesting.T, s *Suite, txs []*types.Transaction, recvConn *Conn) {
+	// Wait for another transaction announcement
+	switch msg := recvConn.ReadAndServe(s.chain, time.Second*8).(type) {
+	case *Transactions:
+		// check to see if any of the failing txs were in the announcement
+		recvTxs := make([]common.Hash, len(*msg))
+		for i, recvTx := range *msg {
+			recvTxs[i] = recvTx.Hash()
+		}
+		badTxs, _ := compareReceivedTxs(recvTxs, txs)
+		if len(badTxs) > 0 {
+			for _, tx := range badTxs {
+				t.Logf("received bad tx: %v", tx)
+			}
+			t.Fatalf("received %d bad txs", len(badTxs))
+		}
+	case *NewPooledTransactionHashes:
+		badTxs, _ := compareReceivedTxs(*msg, txs)
+		if len(badTxs) > 0 {
+			for _, tx := range badTxs {
+				t.Logf("received bad tx: %v", tx)
+			}
+			t.Fatalf("received %d bad txs", len(badTxs))
+		}
 	case *Error:
 		// Transaction should not be announced -> wait for timeout
 		return
@@ -96,6 +157,29 @@ func sendFailingTxWithConns(t *utesting.T, s *Suite, tx *types.Transaction, send
 	}
 }
 
+// compareReceivedTxs compares the received set of txs against the given set of txs,
+// returning both the set received txs that were present within the given txs, and
+// the set of txs that were missing from the set of received txs
+func compareReceivedTxs(recvTxs []common.Hash, txs []*types.Transaction) (present []*types.Transaction, missing []*types.Transaction) {
+	// create a map of the hashes received from node
+	recvHashes := make(map[common.Hash]common.Hash)
+	for _, hash := range recvTxs {
+		recvHashes[hash] = hash
+	}
+
+	// collect present txs and missing txs separately
+	present = make([]*types.Transaction, 0)
+	missing = make([]*types.Transaction, 0)
+	for _, tx := range txs {
+		if _, exists := recvHashes[tx.Hash()]; exists {
+			present = append(present, tx)
+		} else {
+			missing = append(missing, tx)
+		}
+	}
+	return present, missing
+}
+
 func unknownTx(t *utesting.T, s *Suite) *types.Transaction {
 	tx := getNextTxFromChain(t, s)
 	var to common.Address
@@ -103,7 +187,7 @@ func unknownTx(t *utesting.T, s *Suite) *types.Transaction {
 		to = *tx.To()
 	}
 	txNew := types.NewTransaction(tx.Nonce()+1, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data())
-	return signWithFaucet(t, txNew)
+	return signWithFaucet(t, s.chain.chainConfig, txNew)
 }
 
 func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction {
@@ -122,6 +206,30 @@ func getNextTxFromChain(t *utesting.T, s *Suite) *types.Transaction {
 	return tx
 }
 
+func generateTxs(t *utesting.T, s *Suite, numTxs int) (map[common.Hash]common.Hash, []*types.Transaction) {
+	txHashMap := make(map[common.Hash]common.Hash, numTxs)
+	txs := make([]*types.Transaction, numTxs)
+
+	nextTx := getNextTxFromChain(t, s)
+	gas := nextTx.Gas()
+
+	nonce = nonce + 1
+	// generate txs
+	for i := 0; i < numTxs; i++ {
+		tx := generateTx(t, s.chain.chainConfig, nonce, gas)
+		txHashMap[tx.Hash()] = tx.Hash()
+		txs[i] = tx
+		nonce = nonce + 1
+	}
+	return txHashMap, txs
+}
+
+func generateTx(t *utesting.T, chainConfig *params.ChainConfig, nonce uint64, gas uint64) *types.Transaction {
+	var to common.Address
+	tx := types.NewTransaction(nonce, to, big.NewInt(1), gas, big.NewInt(1), []byte{})
+	return signWithFaucet(t, chainConfig, tx)
+}
+
 func getOldTxFromChain(t *utesting.T, s *Suite) *types.Transaction {
 	var tx *types.Transaction
 	for _, blocks := range s.fullChain.blocks[:s.chain.Len()-1] {
@@ -144,7 +252,7 @@ func invalidNonceTx(t *utesting.T, s *Suite) *types.Transaction {
 		to = *tx.To()
 	}
 	txNew := types.NewTransaction(tx.Nonce()-2, to, tx.Value(), tx.Gas(), tx.GasPrice(), tx.Data())
-	return signWithFaucet(t, txNew)
+	return signWithFaucet(t, s.chain.chainConfig, txNew)
 }
 
 func hugeAmount(t *utesting.T, s *Suite) *types.Transaction {
@@ -155,7 +263,7 @@ func hugeAmount(t *utesting.T, s *Suite) *types.Transaction {
 		to = *tx.To()
 	}
 	txNew := types.NewTransaction(tx.Nonce(), to, amount, tx.Gas(), tx.GasPrice(), tx.Data())
-	return signWithFaucet(t, txNew)
+	return signWithFaucet(t, s.chain.chainConfig, txNew)
 }
 
 func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction {
@@ -166,7 +274,7 @@ func hugeGasPrice(t *utesting.T, s *Suite) *types.Transaction {
 		to = *tx.To()
 	}
 	txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), gasPrice, tx.Data())
-	return signWithFaucet(t, txNew)
+	return signWithFaucet(t, s.chain.chainConfig, txNew)
 }
 
 func hugeData(t *utesting.T, s *Suite) *types.Transaction {
@@ -176,11 +284,11 @@ func hugeData(t *utesting.T, s *Suite) *types.Transaction {
 		to = *tx.To()
 	}
 	txNew := types.NewTransaction(tx.Nonce(), to, tx.Value(), tx.Gas(), tx.GasPrice(), largeBuffer(2))
-	return signWithFaucet(t, txNew)
+	return signWithFaucet(t, s.chain.chainConfig, txNew)
 }
 
-func signWithFaucet(t *utesting.T, tx *types.Transaction) *types.Transaction {
-	signer := types.HomesteadSigner{}
+func signWithFaucet(t *utesting.T, chainConfig *params.ChainConfig, tx *types.Transaction) *types.Transaction {
+	signer := types.LatestSigner(chainConfig)
 	signedTx, err := types.SignTx(tx, signer, faucetKey)
 	if err != nil {
 		t.Fatalf("could not sign tx: %v\n", err)

cmd/devp2p/internal/ethtest/types.go

@@ -120,11 +120,20 @@ type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket
 
 func (nb NewPooledTransactionHashes) Code() int { return 24 }
 
+type GetPooledTransactions eth.GetPooledTransactionsPacket
+
+func (gpt GetPooledTransactions) Code() int { return 25 }
+
+type PooledTransactions eth.PooledTransactionsPacket
+
+func (pt PooledTransactions) Code() int { return 26 }
+
 // Conn represents an individual connection with a peer
 type Conn struct {
 	*rlpx.Conn
 	ourKey                 *ecdsa.PrivateKey
-	ethProtocolVersion uint
+	negotiatedProtoVersion uint
+	ourHighestProtoVersion uint
 	caps                   []p2p.Cap
 }
 
@@ -162,6 +171,10 @@ func (c *Conn) Read() Message {
 		msg = new(Transactions)
 	case (NewPooledTransactionHashes{}).Code():
 		msg = new(NewPooledTransactionHashes)
+	case (GetPooledTransactions{}.Code()):
+		msg = new(GetPooledTransactions)
+	case (PooledTransactions{}.Code()):
+		msg = new(PooledTransactions)
 	default:
 		return errorf("invalid message code: %d", code)
 	}
@@ -177,8 +190,7 @@ func (c *Conn) Read() Message {
 func (c *Conn) ReadAndServe(chain *Chain, timeout time.Duration) Message {
 	start := time.Now()
 	for time.Since(start) < timeout {
-		timeout := time.Now().Add(10 * time.Second)
-		c.SetReadDeadline(timeout)
+		c.SetReadDeadline(time.Now().Add(5 * time.Second))
 		switch msg := c.Read().(type) {
 		case *Ping:
 			c.Write(&Pong{})
@@ -236,7 +248,7 @@ func (c *Conn) handshake(t *utesting.T) Message {
 			c.SetSnappy(true)
 		}
 		c.negotiateEthProtocol(msg.Caps)
-		if c.ethProtocolVersion == 0 {
+		if c.negotiatedProtoVersion == 0 {
 			t.Fatalf("unexpected eth protocol version")
 		}
 		return msg
@@ -254,11 +266,11 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
 		if capability.Name != "eth" {
 			continue
 		}
-		if capability.Version > highestEthVersion && capability.Version <= 65 {
+		if capability.Version > highestEthVersion && capability.Version <= c.ourHighestProtoVersion {
 			highestEthVersion = capability.Version
 		}
 	}
-	c.ethProtocolVersion = highestEthVersion
+	c.negotiatedProtoVersion = highestEthVersion
 }
 
 // statusExchange performs a `Status` message exchange with the given
@@ -273,14 +285,15 @@ loop:
 	for {
 		switch msg := c.Read().(type) {
 		case *Status:
-			if msg.Head != chain.blocks[chain.Len()-1].Hash() {
-				t.Fatalf("wrong head block in status: %s", msg.Head.String())
+			if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want {
+				t.Fatalf("wrong head block in status, want:  %#x (block %d) have %#x",
+					want, chain.blocks[chain.Len()-1].NumberU64(), have)
 			}
-			if msg.TD.Cmp(chain.TD(chain.Len())) != 0 {
-				t.Fatalf("wrong TD in status: %v", msg.TD)
+			if have, want := msg.TD.Cmp(chain.TD(chain.Len())), 0; have != want {
+				t.Fatalf("wrong TD in status: have %v want %v", have, want)
 			}
-			if !reflect.DeepEqual(msg.ForkID, chain.ForkID()) {
-				t.Fatalf("wrong fork ID in status: %v", msg.ForkID)
+			if have, want := msg.ForkID, chain.ForkID(); !reflect.DeepEqual(have, want) {
+				t.Fatalf("wrong fork ID in status: have %v, want %v", have, want)
 			}
 			message = msg
 			break loop
@@ -294,13 +307,13 @@ loop:
 		}
 	}
 	// make sure eth protocol version is set for negotiation
-	if c.ethProtocolVersion == 0 {
+	if c.negotiatedProtoVersion == 0 {
 		t.Fatalf("eth protocol version must be set in Conn")
 	}
 	if status == nil {
 		// write status message to client
 		status = &Status{
-			ProtocolVersion: uint32(c.ethProtocolVersion),
+			ProtocolVersion: uint32(c.negotiatedProtoVersion),
 			NetworkID:       chain.chainConfig.ChainID.Uint64(),
 			TD:              chain.TD(chain.Len()),
 			Head:            chain.blocks[chain.Len()-1].Hash(),
@@ -321,8 +334,11 @@ loop:
 func (c *Conn) waitForBlock(block *types.Block) error {
 	defer c.SetReadDeadline(time.Time{})
 
-	timeout := time.Now().Add(20 * time.Second)
-	c.SetReadDeadline(timeout)
+	c.SetReadDeadline(time.Now().Add(20 * time.Second))
+	// note: if the node has not yet imported the block, it will respond
+	// to the GetBlockHeaders request with an empty BlockHeaders response,
+	// so the GetBlockHeaders request must be sent again until the BlockHeaders
+	// response contains the desired header.
 	for {
 		req := &GetBlockHeaders{Origin: eth.HashOrNumber{Hash: block.Hash()}, Amount: 1}
 		if err := c.Write(req); err != nil {
@@ -330,9 +346,11 @@ func (c *Conn) waitForBlock(block *types.Block) error {
 		}
 		switch msg := c.Read().(type) {
 		case *BlockHeaders:
-			if len(*msg) > 0 {
+			for _, header := range *msg {
+				if header.Number.Uint64() == block.NumberU64() {
 					return nil
 				}
+			}
 			time.Sleep(100 * time.Millisecond)
 		default:
 			return fmt.Errorf("invalid message: %s", pretty.Sdump(msg))

cmd/devp2p/nodeset.go

@@ -71,6 +71,7 @@ func writeNodesJSON(file string, nodes nodeSet) {
 	}
 }
 
+// nodes returns the node records contained in the set.
 func (ns nodeSet) nodes() []*enode.Node {
 	result := make([]*enode.Node, 0, len(ns))
 	for _, n := range ns {
@@ -83,12 +84,37 @@ func (ns nodeSet) nodes() []*enode.Node {
 	return result
 }
 
+// add ensures the given nodes are present in the set.
 func (ns nodeSet) add(nodes ...*enode.Node) {
 	for _, n := range nodes {
-		ns[n.ID()] = nodeJSON{Seq: n.Seq(), N: n}
+		v := ns[n.ID()]
+		v.N = n
+		v.Seq = n.Seq()
+		ns[n.ID()] = v
 	}
 }
 
+// topN returns the top n nodes by score as a new set.
+func (ns nodeSet) topN(n int) nodeSet {
+	if n >= len(ns) {
+		return ns
+	}
+
+	byscore := make([]nodeJSON, 0, len(ns))
+	for _, v := range ns {
+		byscore = append(byscore, v)
+	}
+	sort.Slice(byscore, func(i, j int) bool {
+		return byscore[i].Score >= byscore[j].Score
+	})
+	result := make(nodeSet, n)
+	for _, v := range byscore[:n] {
+		result[v.N.ID()] = v
+	}
+	return result
+}
+
+// verify performs integrity checks on the node set.
 func (ns nodeSet) verify() error {
 	for id, n := range ns {
 		if n.N.ID() != id {

cmd/devp2p/nodesetcmd.go

@@ -17,8 +17,12 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"net"
+	"sort"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/core/forkid"
@@ -60,25 +64,64 @@ func nodesetInfo(ctx *cli.Context) error {
 
 	ns := loadNodesJSON(ctx.Args().First())
 	fmt.Printf("Set contains %d nodes.\n", len(ns))
+	showAttributeCounts(ns)
 	return nil
 }
 
+// showAttributeCounts prints the distribution of ENR attributes in a node set.
+func showAttributeCounts(ns nodeSet) {
+	attrcount := make(map[string]int)
+	var attrlist []interface{}
+	for _, n := range ns {
+		r := n.N.Record()
+		attrlist = r.AppendElements(attrlist[:0])[1:]
+		for i := 0; i < len(attrlist); i += 2 {
+			key := attrlist[i].(string)
+			attrcount[key]++
+		}
+	}
+
+	var keys []string
+	var maxlength int
+	for key := range attrcount {
+		keys = append(keys, key)
+		if len(key) > maxlength {
+			maxlength = len(key)
+		}
+	}
+	sort.Strings(keys)
+	fmt.Println("ENR attribute counts:")
+	for _, key := range keys {
+		fmt.Printf("%s%s: %d\n", strings.Repeat(" ", maxlength-len(key)+1), key, attrcount[key])
+	}
+}
+
 func nodesetFilter(ctx *cli.Context) error {
 	if ctx.NArg() < 1 {
 		return fmt.Errorf("need nodes file as argument")
 	}
-	ns := loadNodesJSON(ctx.Args().First())
+	// Parse -limit.
+	limit, err := parseFilterLimit(ctx.Args().Tail())
+	if err != nil {
+		return err
+	}
+	// Parse the filters.
 	filter, err := andFilter(ctx.Args().Tail())
 	if err != nil {
 		return err
 	}
 
+	// Load nodes and apply filters.
+	ns := loadNodesJSON(ctx.Args().First())
 	result := make(nodeSet)
 	for id, n := range ns {
 		if filter(n) {
 			result[id] = n
 		}
 	}
+	if limit >= 0 {
+		result = result.topN(limit)
+	}
 	writeNodesJSON("-", result)
 	return nil
 }
@@ -91,6 +134,7 @@ type nodeFilterC struct {
 }
 
 var filterFlags = map[string]nodeFilterC{
+	"-limit":       {1, trueFilter}, // needed to skip over -limit
 	"-ip":          {1, ipFilter},
 	"-min-age":     {1, minAgeFilter},
 	"-eth-network": {1, ethFilter},
@@ -98,6 +142,7 @@ var filterFlags = map[string]nodeFilterC{
 	"-snap":        {0, snapFilter},
 }
 
+// parseFilters parses nodeFilters from args.
 func parseFilters(args []string) ([]nodeFilter, error) {
 	var filters []nodeFilter
 	for len(args) > 0 {
@@ -118,6 +163,26 @@ func parseFilters(args []string) ([]nodeFilter, error) {
 	return filters, nil
 }
 
+// parseFilterLimit parses the -limit option in args. It returns -1 if there is no limit.
+func parseFilterLimit(args []string) (int, error) {
+	limit := -1
+	for i, arg := range args {
+		if arg == "-limit" {
+			if i == len(args)-1 {
+				return -1, errors.New("-limit requires an argument")
+			}
+			n, err := strconv.Atoi(args[i+1])
+			if err != nil {
+				return -1, fmt.Errorf("invalid -limit %q", args[i+1])
+			}
+			limit = n
+		}
+	}
+	return limit, nil
+}
+
+// andFilter parses node filters in args and and returns a single filter that requires all
+// of them to match.
 func andFilter(args []string) (nodeFilter, error) {
 	checks, err := parseFilters(args)
 	if err != nil {
@@ -134,6 +199,10 @@ func andFilter(args []string) (nodeFilter, error) {
 	return f, nil
 }
 
+func trueFilter(args []string) (nodeFilter, error) {
+	return func(n nodeJSON) bool { return true }, nil
+}
+
 func ipFilter(args []string) (nodeFilter, error) {
 	_, cidr, err := net.ParseCIDR(args[0])
 	if err != nil {
@@ -173,7 +242,7 @@ func ethFilter(args []string) (nodeFilter, error) {
 	f := func(n nodeJSON) bool {
 		var eth struct {
 			ForkID forkid.ID
-			_      []rlp.RawValue `rlp:"tail"`
+			Tail   []rlp.RawValue `rlp:"tail"`
 		}
 		if n.N.Load(enr.WithEntry("eth", &eth)) != nil {
 			return false
@@ -186,7 +255,7 @@ func ethFilter(args []string) (nodeFilter, error) {
 func lesFilter(args []string) (nodeFilter, error) {
 	f := func(n nodeJSON) bool {
 		var les struct {
-			_ []rlp.RawValue `rlp:"tail"`
+			Tail []rlp.RawValue `rlp:"tail"`
 		}
 		return n.N.Load(enr.WithEntry("les", &les)) == nil
 	}
@@ -196,7 +265,7 @@ func lesFilter(args []string) (nodeFilter, error) {
 func snapFilter(args []string) (nodeFilter, error) {
 	f := func(n nodeJSON) bool {
 		var snap struct {
-			_ []rlp.RawValue `rlp:"tail"`
+			Tail []rlp.RawValue `rlp:"tail"`
 		}
 		return n.N.Load(enr.WithEntry("snap", &snap)) == nil
 	}

cmd/devp2p/rlpxcmd.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/internal/utesting"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/rlpx"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -98,5 +99,10 @@ func rlpxEthTest(ctx *cli.Context) error {
 	if err != nil {
 		exit(err)
 	}
+	// check if given node supports eth66, and if so, run eth66 protocol tests as well
+	is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66})
+	if is66Failed {
 		return runTests(ctx, suite.EthTests())
 	}
+	return runTests(ctx, suite.AllEthTests())
+}

cmd/evm/README.md

@@ -256,9 +256,9 @@ Error code: 4
 Another thing that can be done, is to chain invocations:
 ```
 ./evm t8n --input.alloc=./testdata/1/alloc.json --input.txs=./testdata/1/txs.json --input.env=./testdata/1/env.json --output.alloc=stdout | ./evm t8n --input.alloc=stdin --input.env=./testdata/1/env.json --input.txs=./testdata/1/txs.json
-INFO [01-21|22:41:22.963] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
-INFO [01-21|22:41:22.966] rejected tx                              index=0 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
-INFO [01-21|22:41:22.967] rejected tx                              index=1 hash="0557ba…18d673" from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
+INFO [01-21|22:41:22.963] rejected tx                              index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
+INFO [01-21|22:41:22.966] rejected tx                              index=0 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
+INFO [01-21|22:41:22.967] rejected tx                              index=1 hash=0557ba..18d673 from=0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192 error="nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1"
 
 ```
 What happened here, is that we first applied two identical transactions, so the second one was rejected. 

cmd/evm/testdata/8/readme.md

@@ -56,8 +56,8 @@ dir=./testdata/8 \
 If we try to execute it on older rules: 
 ```
 dir=./testdata/8 && ./evm t8n --state.fork=Istanbul --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json 
-INFO [01-21|23:21:51.265] rejected tx                              index=0 hash="d2818d…6ab3da" error="tx type not supported"
-INFO [01-21|23:21:51.265] rejected tx                              index=1 hash="26ea00…81c01b" from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0"
-INFO [01-21|23:21:51.265] rejected tx                              index=2 hash="698d01…369cee" error="tx type not supported"
+INFO [01-21|23:21:51.265] rejected tx                              index=0 hash=d2818d..6ab3da error="tx type not supported"
+INFO [01-21|23:21:51.265] rejected tx                              index=1 hash=26ea00..81c01b from=0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B error="nonce too high: address 0xa94f5374Fce5edBC8E2a8697C15331677e6EbF0B, tx: 1 state: 0"
+INFO [01-21|23:21:51.265] rejected tx                              index=2 hash=698d01..369cee error="tx type not supported"
 ```
 Number `1` and `3` are not applicable, and therefore number `2` has wrong nonce. 

cmd/faucet/faucet.go

@@ -85,6 +85,9 @@ var (
 
 	twitterTokenFlag   = flag.String("twitter.token", "", "Bearer token to authenticate with the v2 Twitter API")
 	twitterTokenV1Flag = flag.String("twitter.token.v1", "", "Bearer token to authenticate with the v1.1 Twitter API")
+
+	goerliFlag  = flag.Bool("goerli", false, "Initializes the faucet with Görli network config")
+	rinkebyFlag = flag.Bool("rinkeby", false, "Initializes the faucet with Rinkeby network config")
 )
 
 var (
@@ -144,13 +147,9 @@ func main() {
 		log.Crit("Failed to render the faucet template", "err", err)
 	}
 	// Load and parse the genesis block requested by the user
-	blob, err := ioutil.ReadFile(*genesisFlag)
+	genesis, err := getGenesis(genesisFlag, *goerliFlag, *rinkebyFlag)
 	if err != nil {
-		log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err)
-	}
-	genesis := new(core.Genesis)
-	if err = json.Unmarshal(blob, genesis); err != nil {
-		log.Crit("Failed to parse genesis block json", "err", err)
+		log.Crit("Failed to parse genesis config", "err", err)
 	}
 	// Convert the bootnodes to internal enode representations
 	var enodes []*enode.Node
@@ -162,7 +161,8 @@ func main() {
 		}
 	}
 	// Load up the account key and decrypt its password
-	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
+	blob, err := ioutil.ReadFile(*accPassFlag)
+	if err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
 	pass := strings.TrimSuffix(string(blob), "\n")
@@ -884,3 +884,19 @@ func authNoAuth(url string) (string, string, common.Address, error) {
 	}
 	return address.Hex() + "@noauth", "", address, nil
 }
+
+// getGenesis returns a genesis based on input args
+func getGenesis(genesisFlag *string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) {
+	switch {
+	case genesisFlag != nil:
+		var genesis core.Genesis
+		err := common.LoadJSON(*genesisFlag, &genesis)
+		return &genesis, err
+	case goerliFlag:
+		return core.DefaultGoerliGenesisBlock(), nil
+	case rinkebyFlag:
+		return core.DefaultRinkebyGenesisBlock(), nil
+	default:
+		return nil, fmt.Errorf("no genesis flag provided")
+	}
+}

cmd/geth/chaincmd.go

@@ -31,11 +31,8 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/eth/downloader"
-	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/ethereum/go-ethereum/trie"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -62,7 +59,11 @@ It expects the genesis file as argument.`,
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
 		Flags: []cli.Flag{
-			utils.DataDirFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -150,27 +151,6 @@ be gzipped.`,
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The export-preimages command export hash preimages to an RLP encoded stream`,
-	}
-	copydbCommand = cli.Command{
-		Action:    utils.MigrateFlags(copyDb),
-		Name:      "copydb",
-		Usage:     "Create a local chain from a target chaindata folder",
-		ArgsUsage: "<sourceChaindataDir>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-			utils.CacheFlag,
-			utils.SyncModeFlag,
-			utils.FakePoWFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.RinkebyFlag,
-			utils.TxLookupLimitFlag,
-			utils.GoerliFlag,
-			utils.YoloV3Flag,
-		},
-		Category: "BLOCKCHAIN COMMANDS",
-		Description: `
-The first argument must be the directory containing the blockchain to download from`,
 	}
 	dumpCommand = cli.Command{
 		Action:    utils.MigrateFlags(dump),
@@ -216,7 +196,7 @@ func initGenesis(ctx *cli.Context) error {
 	defer stack.Close()
 
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		chaindb, err := stack.OpenDatabase(name, 0, 0, "")
+		chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}
@@ -231,6 +211,7 @@ func initGenesis(ctx *cli.Context) error {
 }
 
 func dumpGenesis(ctx *cli.Context) error {
+	// TODO(rjl493456442) support loading from the custom datadir
 	genesis := utils.MakeGenesis(ctx)
 	if genesis == nil {
 		genesis = core.DefaultGenesisBlock()
@@ -253,7 +234,7 @@ func importChain(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, db := utils.MakeChain(ctx, stack, false)
+	chain, db := utils.MakeChain(ctx, stack)
 	defer db.Close()
 
 	// Start periodically gathering memory profiles
@@ -329,7 +310,7 @@ func exportChain(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, _ := utils.MakeChain(ctx, stack, true)
+	chain, _ := utils.MakeChain(ctx, stack)
 	start := time.Now()
 
 	var err error
@@ -346,6 +327,9 @@ func exportChain(ctx *cli.Context) error {
 		if first < 0 || last < 0 {
 			utils.Fatalf("Export error: block number must be greater than 0\n")
 		}
+		if head := chain.CurrentFastBlock(); uint64(last) > head.NumberU64() {
+			utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.NumberU64())
+		}
 		err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last))
 	}
 
@@ -365,7 +349,7 @@ func importPreimages(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	db := utils.MakeChainDatabase(ctx, stack)
+	db := utils.MakeChainDatabase(ctx, stack, false)
 	start := time.Now()
 
 	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
@@ -384,7 +368,7 @@ func exportPreimages(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	db := utils.MakeChainDatabase(ctx, stack)
+	db := utils.MakeChainDatabase(ctx, stack, true)
 	start := time.Now()
 
 	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
@@ -394,81 +378,31 @@ func exportPreimages(ctx *cli.Context) error {
 	return nil
 }
 
-func copyDb(ctx *cli.Context) error {
-	// Ensure we have a source chain directory to copy
-	if len(ctx.Args()) < 1 {
-		utils.Fatalf("Source chaindata directory path argument missing")
-	}
-	if len(ctx.Args()) < 2 {
-		utils.Fatalf("Source ancient chain directory path argument missing")
-	}
-	// Initialize a new chain for the running node to sync into
-	stack, _ := makeConfigNode(ctx)
-	defer stack.Close()
-
-	chain, chainDb := utils.MakeChain(ctx, stack, false)
-	syncMode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
-
-	var syncBloom *trie.SyncBloom
-	if syncMode == downloader.FastSync {
-		syncBloom = trie.NewSyncBloom(uint64(ctx.GlobalInt(utils.CacheFlag.Name)/2), chainDb)
-	}
-	dl := downloader.New(0, chainDb, syncBloom, new(event.TypeMux), chain, nil, nil)
-
-	// Create a source peer to satisfy downloader requests from
-	db, err := rawdb.NewLevelDBDatabaseWithFreezer(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name)/2, 256, ctx.Args().Get(1), "")
-	if err != nil {
-		return err
-	}
-	hc, err := core.NewHeaderChain(db, chain.Config(), chain.Engine(), func() bool { return false })
-	if err != nil {
-		return err
-	}
-	peer := downloader.NewFakePeer("local", db, hc, dl)
-	if err = dl.RegisterPeer("local", 63, peer); err != nil {
-		return err
-	}
-	// Synchronise with the simulated peer
-	start := time.Now()
-
-	currentHeader := hc.CurrentHeader()
-	if err = dl.Synchronise("local", currentHeader.Hash(), hc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()), syncMode); err != nil {
-		return err
-	}
-	for dl.Synchronising() {
-		time.Sleep(10 * time.Millisecond)
-	}
-	fmt.Printf("Database copy done in %v\n", time.Since(start))
-
-	// Compact the entire database to remove any sync overhead
-	start = time.Now()
-	fmt.Println("Compacting entire database...")
-	if err = db.Compact(nil, nil); err != nil {
-		utils.Fatalf("Compaction failed: %v", err)
-	}
-	fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
-	return nil
-}
-
 func dump(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, chainDb := utils.MakeChain(ctx, stack, true)
-	defer chainDb.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
 	for _, arg := range ctx.Args() {
-		var block *types.Block
+		var header *types.Header
 		if hashish(arg) {
-			block = chain.GetBlockByHash(common.HexToHash(arg))
-		} else {
-			num, _ := strconv.Atoi(arg)
-			block = chain.GetBlockByNumber(uint64(num))
+			hash := common.HexToHash(arg)
+			number := rawdb.ReadHeaderNumber(db, hash)
+			if number != nil {
+				header = rawdb.ReadHeader(db, hash, *number)
 			}
-		if block == nil {
+		} else {
+			number, _ := strconv.Atoi(arg)
+			hash := rawdb.ReadCanonicalHash(db, uint64(number))
+			if hash != (common.Hash{}) {
+				header = rawdb.ReadHeader(db, hash, uint64(number))
+			}
+		}
+		if header == nil {
 			fmt.Println("{}")
 			utils.Fatalf("block not found")
 		} else {
-			state, err := state.New(block.Root(), state.NewDatabase(chainDb), nil)
+			state, err := state.New(header.Root, state.NewDatabase(db), nil)
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}

cmd/geth/config.go

@@ -28,6 +28,7 @@ import (
 	"gopkg.in/urfave/cli.v1"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/metrics"
@@ -146,7 +147,17 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.GlobalIsSet(utils.OverrideBerlinFlag.Name) {
 		cfg.Eth.OverrideBerlin = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideBerlinFlag.Name))
 	}
-	backend := utils.RegisterEthService(stack, &cfg.Eth)
+	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
+
+	// Configure catalyst.
+	if ctx.GlobalBool(utils.CatalystFlag.Name) {
+		if eth == nil {
+			utils.Fatalf("Catalyst does not work in light client mode.")
+		}
+		if err := catalyst.Register(stack, eth); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
 
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {

cmd/geth/dao_test.go

@@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "geth", "chaindata")
-	db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "")
+	db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
 	}

cmd/geth/dbcmd.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -28,9 +30,8 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/ethereum/go-ethereum/trie"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -59,52 +60,141 @@ Remove blockchain and state databases`,
 			dbGetCmd,
 			dbDeleteCmd,
 			dbPutCmd,
+			dbGetSlotsCmd,
+			dbDumpFreezerIndex,
 		},
 	}
 	dbInspectCmd = cli.Command{
 		Action:    utils.MigrateFlags(inspect),
 		Name:      "inspect",
 		ArgsUsage: "<prefix> <start>",
-
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
 	dbStatCmd = cli.Command{
-		Action: dbStats,
+		Action: utils.MigrateFlags(dbStats),
 		Name:   "stats",
 		Usage:  "Print leveldb statistics",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
 	}
 	dbCompactCmd = cli.Command{
-		Action: dbCompact,
+		Action: utils.MigrateFlags(dbCompact),
 		Name:   "compact",
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+			utils.CacheFlag,
+			utils.CacheDatabaseFlag,
+		},
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
 	}
 	dbGetCmd = cli.Command{
-		Action:      dbGet,
+		Action:    utils.MigrateFlags(dbGet),
 		Name:      "get",
 		Usage:     "Show the value of a database key",
 		ArgsUsage: "<hex-encoded key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = cli.Command{
-		Action:    dbDelete,
+		Action:    utils.MigrateFlags(dbDelete),
 		Name:      "delete",
 		Usage:     "Delete a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
 	dbPutCmd = cli.Command{
-		Action:    dbPut,
+		Action:    utils.MigrateFlags(dbPut),
 		Name:      "put",
 		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
+	dbGetSlotsCmd = cli.Command{
+		Action:    utils.MigrateFlags(dbDumpTrie),
+		Name:      "dumptrie",
+		Usage:     "Show the storage key/values of a given storage trie",
+		ArgsUsage: "<hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
+		Description: "This command looks up the specified database key from the database.",
+	}
+	dbDumpFreezerIndex = cli.Command{
+		Action:    utils.MigrateFlags(freezerInspect),
+		Name:      "freezer-index",
+		Usage:     "Dump out the index of a given freezer type",
+		ArgsUsage: "<type> <start (int)> <end (int)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.YoloV3Flag,
+		},
+		Description: "This command displays information about the freezer index.",
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -192,10 +282,10 @@ func inspect(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	_, chainDb := utils.MakeChain(ctx, stack, true)
-	defer chainDb.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
 
-	return rawdb.InspectDatabase(chainDb, prefix, start)
+	return rawdb.InspectDatabase(db, prefix, start)
 }
 
 func showLeveldbStats(db ethdb.Stater) {
@@ -214,49 +304,33 @@ func showLeveldbStats(db ethdb.Stater) {
 func dbStats(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	path := stack.ResolvePath("chaindata")
-	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
-		options.ReadOnly = true
-	})
-	if err != nil {
-		return err
-	}
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
 	showLeveldbStats(db)
-	err = db.Close()
-	if err != nil {
-		log.Info("Close err", "error", err)
-	}
 	return nil
 }
 
 func dbCompact(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	path := stack.ResolvePath("chaindata")
-	cache := ctx.GlobalInt(utils.CacheFlag.Name) * ctx.GlobalInt(utils.CacheDatabaseFlag.Name) / 100
-	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
-		options.OpenFilesCacheCapacity = utils.MakeDatabaseHandles()
-		options.BlockCacheCapacity = cache / 2 * opt.MiB
-		options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally
-	})
-	if err != nil {
-		return err
-	}
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
+
+	log.Info("Stats before compaction")
 	showLeveldbStats(db)
+
 	log.Info("Triggering compaction")
-	err = db.Compact(nil, nil)
-	if err != nil {
+	if err := db.Compact(nil, nil); err != nil {
 		log.Info("Compact err", "error", err)
-	}
-	showLeveldbStats(db)
-	log.Info("Closing db")
-	err = db.Close()
-	if err != nil {
-		log.Info("Close err", "error", err)
-	}
-	log.Info("Exiting")
 		return err
 	}
+	log.Info("Stats after compaction")
+	showLeveldbStats(db)
+	return nil
+}
 
 // dbGet shows the value of a given database key
 func dbGet(ctx *cli.Context) error {
@@ -265,14 +339,10 @@ func dbGet(ctx *cli.Context) error {
 	}
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	path := stack.ResolvePath("chaindata")
-	db, err := leveldb.NewCustom(path, "", func(options *opt.Options) {
-		options.ReadOnly = true
-	})
-	if err != nil {
-		return err
-	}
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
 	defer db.Close()
+
 	key, err := hexutil.Decode(ctx.Args().Get(0))
 	if err != nil {
 		log.Info("Could not decode the key", "error", err)
@@ -283,7 +353,7 @@ func dbGet(ctx *cli.Context) error {
 		log.Info("Get operation failed", "error", err)
 		return err
 	}
-	fmt.Printf("key %#x:\n\t%#x\n", key, data)
+	fmt.Printf("key %#x: %#x\n", key, data)
 	return nil
 }
 
@@ -294,13 +364,19 @@ func dbDelete(ctx *cli.Context) error {
 	}
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	db := utils.MakeChainDatabase(ctx, stack)
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
 	defer db.Close()
+
 	key, err := hexutil.Decode(ctx.Args().Get(0))
 	if err != nil {
 		log.Info("Could not decode the key", "error", err)
 		return err
 	}
+	data, err := db.Get(key)
+	if err == nil {
+		fmt.Printf("Previous value: %#x\n", data)
+	}
 	if err = db.Delete(key); err != nil {
 		log.Info("Delete operation returned an error", "error", err)
 		return err
@@ -315,8 +391,10 @@ func dbPut(ctx *cli.Context) error {
 	}
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	db := utils.MakeChainDatabase(ctx, stack)
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
 	defer db.Close()
+
 	var (
 		key   []byte
 		value []byte
@@ -335,7 +413,97 @@ func dbPut(ctx *cli.Context) error {
 	}
 	data, err = db.Get(key)
 	if err == nil {
-		fmt.Printf("Previous value:\n%#x\n", data)
+		fmt.Printf("Previous value: %#x\n", data)
 	}
 	return db.Put(key, value)
 }
+
+// dbDumpTrie shows the key-value slots of a given storage trie
+func dbDumpTrie(ctx *cli.Context) error {
+	if ctx.NArg() < 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	var (
+		root  []byte
+		start []byte
+		max   = int64(-1)
+		err   error
+	)
+	if root, err = hexutil.Decode(ctx.Args().Get(0)); err != nil {
+		log.Info("Could not decode the root", "error", err)
+		return err
+	}
+	stRoot := common.BytesToHash(root)
+	if ctx.NArg() >= 2 {
+		if start, err = hexutil.Decode(ctx.Args().Get(1)); err != nil {
+			log.Info("Could not decode the seek position", "error", err)
+			return err
+		}
+	}
+	if ctx.NArg() >= 3 {
+		if max, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
+			log.Info("Could not decode the max count", "error", err)
+			return err
+		}
+	}
+	theTrie, err := trie.New(stRoot, trie.NewDatabase(db))
+	if err != nil {
+		return err
+	}
+	var count int64
+	it := trie.NewIterator(theTrie.NodeIterator(start))
+	for it.Next() {
+		if max > 0 && count == max {
+			fmt.Printf("Exiting after %d values\n", count)
+			break
+		}
+		fmt.Printf("  %d. key %#x: %#x\n", count, it.Key, it.Value)
+		count++
+	}
+	return it.Err
+}
+
+func freezerInspect(ctx *cli.Context) error {
+	var (
+		start, end    int64
+		disableSnappy bool
+		err           error
+	)
+	if ctx.NArg() < 3 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	kind := ctx.Args().Get(0)
+	if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok {
+		var options []string
+		for opt := range rawdb.FreezerNoSnappy {
+			options = append(options, opt)
+		}
+		sort.Strings(options)
+		return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options)
+	} else {
+		disableSnappy = noSnap
+	}
+	if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil {
+		log.Info("Could read start-param", "error", err)
+		return err
+	}
+	if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
+		log.Info("Could read count param", "error", err)
+		return err
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
+	log.Info("Opening freezer", "location", path, "name", kind)
+	if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy); err != nil {
+		return err
+	} else {
+		f.DumpIndex(start, end)
+	}
+	return nil
+}

cmd/geth/main.go

@@ -19,9 +19,7 @@ package main
 
 import (
 	"fmt"
-	"math"
 	"os"
-	godebug "runtime/debug"
 	"sort"
 	"strconv"
 	"strings"
@@ -41,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	gopsutil "github.com/shirou/gopsutil/mem"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -153,7 +150,9 @@ var (
 		utils.GpoMaxGasPriceFlag,
 		utils.EWASMInterpreterFlag,
 		utils.EVMInterpreterFlag,
+		utils.MinerNotifyFullFlag,
 		configFileFlag,
+		utils.CatalystFlag,
 	}
 
 	rpcFlags = []cli.Flag{
@@ -213,7 +212,6 @@ func init() {
 		exportCommand,
 		importPreimagesCommand,
 		exportPreimagesCommand,
-		copydbCommand,
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
@@ -304,25 +302,6 @@ func prepare(ctx *cli.Context) {
 		log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128)
 		ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128))
 	}
-	// Cap the cache allowance and tune the garbage collector
-	mem, err := gopsutil.VirtualMemory()
-	if err == nil {
-		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
-			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
-			mem.Total = 2 * 1024 * 1024 * 1024
-		}
-		allowance := int(mem.Total / 1024 / 1024 / 3)
-		if cache := ctx.GlobalInt(utils.CacheFlag.Name); cache > allowance {
-			log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
-			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(allowance))
-		}
-	}
-	// Ensure Go's GC ignores the database cache for trigger percentage
-	cache := ctx.GlobalInt(utils.CacheFlag.Name)
-	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
-
-	log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
-	godebug.SetGCPercent(int(gogc))
 
 	// Start metrics export if enabled
 	utils.SetupMetrics(ctx)

cmd/geth/snapshot.go

@@ -57,6 +57,7 @@ var (
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.RinkebyFlag,
 					utils.GoerliFlag,
@@ -86,6 +87,7 @@ the trie clean cache with default directory will be deleted.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.RinkebyFlag,
 					utils.GoerliFlag,
@@ -105,6 +107,7 @@ In other words, this command does the snapshot to trie conversion.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.RinkebyFlag,
 					utils.GoerliFlag,
@@ -126,6 +129,7 @@ It's also usable without snapshot enabled.
 				Category:  "MISCELLANEOUS COMMANDS",
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
+					utils.AncientFlag,
 					utils.RopstenFlag,
 					utils.RinkebyFlag,
 					utils.GoerliFlag,
@@ -148,12 +152,10 @@ func pruneState(ctx *cli.Context) error {
 	stack, config := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, chaindb := utils.MakeChain(ctx, stack, true)
-	defer chaindb.Close()
-
-	pruner, err := pruner.NewPruner(chaindb, chain.CurrentBlock().Header(), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
+	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name))
 	if err != nil {
-		log.Error("Failed to open snapshot tree", "error", err)
+		log.Error("Failed to open snapshot tree", "err", err)
 		return err
 	}
 	if ctx.NArg() > 1 {
@@ -164,12 +166,12 @@ func pruneState(ctx *cli.Context) error {
 	if ctx.NArg() == 1 {
 		targetRoot, err = parseRoot(ctx.Args()[0])
 		if err != nil {
-			log.Error("Failed to resolve state root", "error", err)
+			log.Error("Failed to resolve state root", "err", err)
 			return err
 		}
 	}
 	if err = pruner.Prune(targetRoot); err != nil {
-		log.Error("Failed to prune state", "error", err)
+		log.Error("Failed to prune state", "err", err)
 		return err
 	}
 	return nil
@@ -179,31 +181,34 @@ func verifyState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, chaindb := utils.MakeChain(ctx, stack, true)
-	defer chaindb.Close()
-
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, chain.CurrentBlock().Root(), false, false, false)
+	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	headBlock := rawdb.ReadHeadBlock(chaindb)
+	if headBlock == nil {
+		log.Error("Failed to load head block")
+		return errors.New("no head block")
+	}
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false)
 	if err != nil {
-		log.Error("Failed to open snapshot tree", "error", err)
+		log.Error("Failed to open snapshot tree", "err", err)
 		return err
 	}
 	if ctx.NArg() > 1 {
 		log.Error("Too many arguments given")
 		return errors.New("too many arguments")
 	}
-	var root = chain.CurrentBlock().Root()
+	var root = headBlock.Root()
 	if ctx.NArg() == 1 {
 		root, err = parseRoot(ctx.Args()[0])
 		if err != nil {
-			log.Error("Failed to resolve state root", "error", err)
+			log.Error("Failed to resolve state root", "err", err)
 			return err
 		}
 	}
 	if err := snaptree.Verify(root); err != nil {
-		log.Error("Failed to verfiy state", "error", err)
+		log.Error("Failed to verfiy state", "root", root, "err", err)
 		return err
 	}
-	log.Info("Verified the state")
+	log.Info("Verified the state", "root", root)
 	return nil
 }
 
@@ -214,19 +219,16 @@ func traverseState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, chaindb := utils.MakeChain(ctx, stack, true)
-	defer chaindb.Close()
-
+	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	headBlock := rawdb.ReadHeadBlock(chaindb)
+	if headBlock == nil {
+		log.Error("Failed to load head block")
+		return errors.New("no head block")
+	}
 	if ctx.NArg() > 1 {
 		log.Error("Too many arguments given")
 		return errors.New("too many arguments")
 	}
-	// Use the HEAD root as the default
-	head := chain.CurrentBlock()
-	if head == nil {
-		log.Error("Head block is missing")
-		return errors.New("head block is missing")
-	}
 	var (
 		root common.Hash
 		err  error
@@ -234,18 +236,18 @@ func traverseState(ctx *cli.Context) error {
 	if ctx.NArg() == 1 {
 		root, err = parseRoot(ctx.Args()[0])
 		if err != nil {
-			log.Error("Failed to resolve state root", "error", err)
+			log.Error("Failed to resolve state root", "err", err)
 			return err
 		}
 		log.Info("Start traversing the state", "root", root)
 	} else {
-		root = head.Root()
-		log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
+		root = headBlock.Root()
+		log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
 	}
 	triedb := trie.NewDatabase(chaindb)
 	t, err := trie.NewSecure(root, triedb)
 	if err != nil {
-		log.Error("Failed to open trie", "root", root, "error", err)
+		log.Error("Failed to open trie", "root", root, "err", err)
 		return err
 	}
 	var (
@@ -260,13 +262,13 @@ func traverseState(ctx *cli.Context) error {
 		accounts += 1
 		var acc state.Account
 		if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil {
-			log.Error("Invalid account encountered during traversal", "error", err)
+			log.Error("Invalid account encountered during traversal", "err", err)
 			return err
 		}
 		if acc.Root != emptyRoot {
 			storageTrie, err := trie.NewSecure(acc.Root, triedb)
 			if err != nil {
-				log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
+				log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
 				return err
 			}
 			storageIter := trie.NewIterator(storageTrie.NodeIterator(nil))
@@ -274,7 +276,7 @@ func traverseState(ctx *cli.Context) error {
 				slots += 1
 			}
 			if storageIter.Err != nil {
-				log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Err)
+				log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err)
 				return storageIter.Err
 			}
 		}
@@ -292,7 +294,7 @@ func traverseState(ctx *cli.Context) error {
 		}
 	}
 	if accIter.Err != nil {
-		log.Error("Failed to traverse state trie", "root", root, "error", accIter.Err)
+		log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err)
 		return accIter.Err
 	}
 	log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
@@ -307,19 +309,16 @@ func traverseRawState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, chaindb := utils.MakeChain(ctx, stack, true)
-	defer chaindb.Close()
-
+	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	headBlock := rawdb.ReadHeadBlock(chaindb)
+	if headBlock == nil {
+		log.Error("Failed to load head block")
+		return errors.New("no head block")
+	}
 	if ctx.NArg() > 1 {
 		log.Error("Too many arguments given")
 		return errors.New("too many arguments")
 	}
-	// Use the HEAD root as the default
-	head := chain.CurrentBlock()
-	if head == nil {
-		log.Error("Head block is missing")
-		return errors.New("head block is missing")
-	}
 	var (
 		root common.Hash
 		err  error
@@ -327,18 +326,18 @@ func traverseRawState(ctx *cli.Context) error {
 	if ctx.NArg() == 1 {
 		root, err = parseRoot(ctx.Args()[0])
 		if err != nil {
-			log.Error("Failed to resolve state root", "error", err)
+			log.Error("Failed to resolve state root", "err", err)
 			return err
 		}
 		log.Info("Start traversing the state", "root", root)
 	} else {
-		root = head.Root()
-		log.Info("Start traversing the state", "root", root, "number", head.NumberU64())
+		root = headBlock.Root()
+		log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
 	}
 	triedb := trie.NewDatabase(chaindb)
 	t, err := trie.NewSecure(root, triedb)
 	if err != nil {
-		log.Error("Failed to open trie", "root", root, "error", err)
+		log.Error("Failed to open trie", "root", root, "err", err)
 		return err
 	}
 	var (
@@ -369,13 +368,13 @@ func traverseRawState(ctx *cli.Context) error {
 			accounts += 1
 			var acc state.Account
 			if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
-				log.Error("Invalid account encountered during traversal", "error", err)
+				log.Error("Invalid account encountered during traversal", "err", err)
 				return errors.New("invalid account")
 			}
 			if acc.Root != emptyRoot {
 				storageTrie, err := trie.NewSecure(acc.Root, triedb)
 				if err != nil {
-					log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
+					log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
 					return errors.New("missing storage trie")
 				}
 				storageIter := storageTrie.NodeIterator(nil)
@@ -398,7 +397,7 @@ func traverseRawState(ctx *cli.Context) error {
 					}
 				}
 				if storageIter.Error() != nil {
-					log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIter.Error())
+					log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error())
 					return storageIter.Error()
 				}
 			}
@@ -417,7 +416,7 @@ func traverseRawState(ctx *cli.Context) error {
 		}
 	}
 	if accIter.Error() != nil {
-		log.Error("Failed to traverse state trie", "root", root, "error", accIter.Error())
+		log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error())
 		return accIter.Error()
 	}
 	log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))

cmd/geth/usage.go

@@ -180,6 +180,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.MiningEnabledFlag,
 			utils.MinerThreadsFlag,
 			utils.MinerNotifyFlag,
+			utils.MinerNotifyFullFlag,
 			utils.MinerGasPriceFlag,
 			utils.MinerGasTargetFlag,
 			utils.MinerGasLimitFlag,
@@ -234,6 +235,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.SnapshotFlag,
 			utils.BloomFilterSizeFlag,
 			cli.HelpFlag,
+			utils.CatalystFlag,
 		},
 	},
 }

cmd/puppeth/module_dashboard.go

@@ -518,6 +518,8 @@ var dashboardMascot = []byte("\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01s\x
 var dashboardDockerfile = `
 FROM mhart/alpine-node:latest
 
+WORKDIR /usr/app
+
 RUN \
 	npm install connect serve-static && \
 	\

cmd/puppeth/ssh.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
@@ -43,6 +44,8 @@ type sshClient struct {
 	logger  log.Logger
 }
 
+const EnvSSHAuthSock = "SSH_AUTH_SOCK"
+
 // dial establishes an SSH connection to a remote node using the current user and
 // the user's configured private RSA key. If that fails, password authentication
 // is fallen back to. server can be a string like user:identity@server:port.
@@ -79,9 +82,19 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 	if username == "" {
 		username = user.Username
 	}
-	// Configure the supported authentication methods (private key and password)
-	var auths []ssh.AuthMethod
 
+	// Configure the supported authentication methods (ssh agent, private key and password)
+	var (
+		auths []ssh.AuthMethod
+		conn  net.Conn
+	)
+	if conn, err = net.Dial("unix", os.Getenv(EnvSSHAuthSock)); err != nil {
+		log.Warn("Unable to dial SSH agent, falling back to private keys", "err", err)
+	} else {
+		client := agent.NewClient(conn)
+		auths = append(auths, ssh.PublicKeysCallback(client.Signers))
+	}
+	if err != nil {
 		path := filepath.Join(user.HomeDir, ".ssh", identity)
 		if buf, err := ioutil.ReadFile(path); err != nil {
 			log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
@@ -111,6 +124,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 			fmt.Println()
 			return string(blob), err
 		}))
+	}
 	// Resolve the IP address of the remote server
 	addr, err := net.LookupHost(hostname)
 	if err != nil {

cmd/utils/diskusage.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !windows
+// +build !windows,!openbsd
 
 package utils
 

cmd/utils/diskusage_openbsd.go

@@ -0,0 +1,43 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build openbsd
+
+package utils
+
+import (
+	"fmt"
+
+	"golang.org/x/sys/unix"
+)
+
+func getFreeDiskSpace(path string) (uint64, error) {
+	var stat unix.Statfs_t
+	if err := unix.Statfs(path, &stat); err != nil {
+		return 0, fmt.Errorf("failed to call Statfs: %v", err)
+	}
+
+	// Available blocks * size per block = available space in bytes
+	var bavail = stat.F_bavail
+	// Not sure if the following check is necessary for OpenBSD
+	if stat.F_bavail < 0 {
+		// FreeBSD can have a negative number of blocks available
+		// because of the grace limit.
+		bavail = 0
+	}
+	//nolint:unconvert
+	return uint64(bavail) * uint64(stat.F_bsize), nil
+}

cmd/utils/flags.go

@@ -22,9 +22,11 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"math/big"
 	"os"
 	"path/filepath"
+	godebug "runtime/debug"
 	"strconv"
 	"strings"
 	"text/tabwriter"
@@ -65,6 +67,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
 	pcsclite "github.com/gballet/go-libpcsclite"
+	gopsutil "github.com/shirou/gopsutil/mem"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -427,6 +430,10 @@ var (
 		Name:  "miner.notify",
 		Usage: "Comma separated HTTP URL list to notify of new work packages",
 	}
+	MinerNotifyFullFlag = cli.BoolFlag{
+		Name:  "miner.notify.full",
+		Usage: "Notify with pending block headers instead of work packages",
+	}
 	MinerGasTargetFlag = cli.Uint64Flag{
 		Name:  "miner.gastarget",
 		Usage: "Target gas floor for mined blocks",
@@ -748,6 +755,11 @@ var (
 		Usage: "External EVM configuration (default = built-in interpreter)",
 		Value: "",
 	}
+
+	CatalystFlag = cli.BoolFlag{
+		Name:  "catalyst",
+		Usage: "Catalyst mode (eth2 integration testing)",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1179,10 +1191,11 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 		cfg.NetRestrict = list
 	}
 
-	if ctx.GlobalBool(DeveloperFlag.Name) {
+	if ctx.GlobalBool(DeveloperFlag.Name) || ctx.GlobalBool(CatalystFlag.Name) {
 		// --dev mode can't use p2p networking.
 		cfg.MaxPeers = 0
-		cfg.ListenAddr = ":0"
+		cfg.ListenAddr = ""
+		cfg.NoDial = true
 		cfg.NoDiscovery = true
 		cfg.DiscoveryV5 = false
 	}
@@ -1359,6 +1372,7 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
 	}
+	cfg.NotifyFull = ctx.GlobalBool(MinerNotifyFullFlag.Name)
 	if ctx.GlobalIsSet(MinerExtraDataFlag.Name) {
 		cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name))
 	}
@@ -1468,6 +1482,26 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setWhitelist(ctx, cfg)
 	setLes(ctx, cfg)
 
+	// Cap the cache allowance and tune the garbage collector
+	mem, err := gopsutil.VirtualMemory()
+	if err == nil {
+		if 32<<(^uintptr(0)>>63) == 32 && mem.Total > 2*1024*1024*1024 {
+			log.Warn("Lowering memory allowance on 32bit arch", "available", mem.Total/1024/1024, "addressable", 2*1024)
+			mem.Total = 2 * 1024 * 1024 * 1024
+		}
+		allowance := int(mem.Total / 1024 / 1024 / 3)
+		if cache := ctx.GlobalInt(CacheFlag.Name); cache > allowance {
+			log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance)
+			ctx.GlobalSet(CacheFlag.Name, strconv.Itoa(allowance))
+		}
+	}
+	// Ensure Go's GC ignores the database cache for trigger percentage
+	cache := ctx.GlobalInt(CacheFlag.Name)
+	gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024)))
+
+	log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
+	godebug.SetGCPercent(int(gogc))
+
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
 	}
@@ -1628,7 +1662,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		if ctx.GlobalIsSet(DataDirFlag.Name) {
 			// Check if we have an already initialized chain and fall back to
 			// that if so. Otherwise we need to generate a new genesis spec.
-			chaindb := MakeChainDatabase(ctx, stack)
+			chaindb := MakeChainDatabase(ctx, stack, false) // TODO (MariusVanDerWijden) make this read only
 			if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
 				cfg.Genesis = nil // fallback to db content
 			}
@@ -1656,23 +1690,21 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) {
 	}
 	if url := params.KnownDNSNetwork(genesis, protocol); url != "" {
 		cfg.EthDiscoveryURLs = []string{url}
-	}
-	if cfg.SyncMode == downloader.SnapSync {
-		if url := params.KnownDNSNetwork(genesis, "snap"); url != "" {
-			cfg.SnapDiscoveryURLs = []string{url}
-		}
+		cfg.SnapDiscoveryURLs = cfg.EthDiscoveryURLs
 	}
 }
 
 // RegisterEthService adds an Ethereum client to the stack.
-func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) ethapi.Backend {
+// The second return value is the full node instance, which may be nil if the
+// node is running as a light client.
+func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend, *eth.Ethereum) {
 	if cfg.SyncMode == downloader.LightSync {
 		backend, err := les.New(stack, cfg)
 		if err != nil {
 			Fatalf("Failed to register the Ethereum service: %v", err)
 		}
 		stack.RegisterAPIs(tracers.APIs(backend.ApiBackend))
-		return backend.ApiBackend
+		return backend.ApiBackend, nil
 	}
 	backend, err := eth.New(stack, cfg)
 	if err != nil {
@@ -1685,7 +1717,7 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) ethapi.Backend
 		}
 	}
 	stack.RegisterAPIs(tracers.APIs(backend.APIBackend))
-	return backend.APIBackend
+	return backend.APIBackend, backend
 }
 
 // RegisterEthStatsService configures the Ethereum Stats daemon and adds it to
@@ -1749,7 +1781,7 @@ func SplitTagsFlag(tagsFlag string) map[string]string {
 }
 
 // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
-func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
+func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database {
 	var (
 		cache   = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 		handles = MakeDatabaseHandles()
@@ -1759,10 +1791,10 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 	)
 	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
 		name := "lightchaindata"
-		chainDb, err = stack.OpenDatabase(name, cache, handles, "")
+		chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly)
 	} else {
 		name := "chaindata"
-		chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "")
+		chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
@@ -1773,6 +1805,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 func MakeGenesis(ctx *cli.Context) *core.Genesis {
 	var genesis *core.Genesis
 	switch {
+	case ctx.GlobalBool(MainnetFlag.Name):
+		genesis = core.DefaultGenesisBlock()
 	case ctx.GlobalBool(RopstenFlag.Name):
 		genesis = core.DefaultRopstenGenesisBlock()
 	case ctx.GlobalBool(RinkebyFlag.Name):
@@ -1788,14 +1822,14 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 }
 
 // MakeChain creates a chain manager from set command line flags.
-func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.BlockChain, chainDb ethdb.Database) {
+func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) {
 	// expecting the last argument to be the genesis file
 	genesis, err := getGenesis(ctx.Args().Get(len(ctx.Args()) - 1))
 	if err != nil {
 		Fatalf("Valid genesis file is required as argument: {}", err)
 	}
 
-	chainDb = MakeChainDatabase(ctx, stack)
+	chainDb = MakeChainDatabase(ctx, stack, false) // TODO(rjl493456442) support read-only database
 	config, _, err := core.SetupGenesisBlock(chainDb, genesis)
 	if err != nil {
 		Fatalf("%v", err)
@@ -1852,12 +1886,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chain *core.B
 		cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
-	var limit *uint64
-	if ctx.GlobalIsSet(TxLookupLimitFlag.Name) && !readOnly {
-		l := ctx.GlobalUint64(TxLookupLimitFlag.Name)
-		limit = &l
-	}
-	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, limit)
+
+	// TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only.
+	// Disable transaction indexing/unindexing by default.
+	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}

common/prque/lazyqueue.go

@@ -55,7 +55,7 @@ type (
 // NewLazyQueue creates a new lazy queue
 func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPriority MaxPriorityCallback, clock mclock.Clock, refreshPeriod time.Duration) *LazyQueue {
 	q := &LazyQueue{
-		popQueue:     newSstack(nil),
+		popQueue:     newSstack(nil, false),
 		setIndex:     setIndex,
 		priority:     priority,
 		maxPriority:  maxPriority,
@@ -71,8 +71,8 @@ func NewLazyQueue(setIndex SetIndexCallback, priority PriorityCallback, maxPrior
 
 // Reset clears the contents of the queue
 func (q *LazyQueue) Reset() {
-	q.queue[0] = newSstack(q.setIndex0)
-	q.queue[1] = newSstack(q.setIndex1)
+	q.queue[0] = newSstack(q.setIndex0, false)
+	q.queue[1] = newSstack(q.setIndex1, false)
 }
 
 // Refresh performs queue re-evaluation if necessary

common/prque/prque.go

@@ -28,7 +28,12 @@ type Prque struct {
 
 // New creates a new priority queue.
 func New(setIndex SetIndexCallback) *Prque {
-	return &Prque{newSstack(setIndex)}
+	return &Prque{newSstack(setIndex, false)}
+}
+
+// NewWrapAround creates a new priority queue with wrap-around priority handling.
+func NewWrapAround(setIndex SetIndexCallback) *Prque {
+	return &Prque{newSstack(setIndex, true)}
 }
 
 // Pushes a value with a given priority into the queue, expanding if necessary.

common/prque/sstack.go

@@ -35,18 +35,20 @@ type sstack struct {
 	size       int
 	capacity   int
 	offset     int
+	wrapAround bool
 
 	blocks [][]*item
 	active []*item
 }
 
 // Creates a new, empty stack.
-func newSstack(setIndex SetIndexCallback) *sstack {
+func newSstack(setIndex SetIndexCallback, wrapAround bool) *sstack {
 	result := new(sstack)
 	result.setIndex = setIndex
 	result.active = make([]*item, blockSize)
 	result.blocks = [][]*item{result.active}
 	result.capacity = blockSize
+	result.wrapAround = wrapAround
 	return result
 }
 
@@ -94,7 +96,11 @@ func (s *sstack) Len() int {
 // Compares the priority of two elements of the stack (higher is first).
 // Required by sort.Interface.
 func (s *sstack) Less(i, j int) bool {
-	return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0
+	a, b := s.blocks[i/blockSize][i%blockSize].priority, s.blocks[j/blockSize][j%blockSize].priority
+	if s.wrapAround {
+		return a-b > 0
+	}
+	return a > b
 }
 
 // Swaps two elements in the stack. Required by sort.Interface.
@@ -110,5 +116,5 @@ func (s *sstack) Swap(i, j int) {
 
 // Resets the stack, effectively clearing its contents.
 func (s *sstack) Reset() {
-	*s = *newSstack(s.setIndex)
+	*s = *newSstack(s.setIndex, false)
 }

common/prque/sstack_test.go

@@ -21,7 +21,7 @@ func TestSstack(t *testing.T) {
 	for i := 0; i < size; i++ {
 		data[i] = &item{rand.Int(), rand.Int63()}
 	}
-	stack := newSstack(nil)
+	stack := newSstack(nil, false)
 	for rep := 0; rep < 2; rep++ {
 		// Push all the data into the stack, pop out every second
 		secs := []*item{}
@@ -55,7 +55,7 @@ func TestSstackSort(t *testing.T) {
 		data[i] = &item{rand.Int(), int64(i)}
 	}
 	// Push all the data into the stack
-	stack := newSstack(nil)
+	stack := newSstack(nil, false)
 	for _, val := range data {
 		stack.Push(val)
 	}
@@ -76,7 +76,7 @@ func TestSstackReset(t *testing.T) {
 	for i := 0; i < size; i++ {
 		data[i] = &item{rand.Int(), rand.Int63()}
 	}
-	stack := newSstack(nil)
+	stack := newSstack(nil, false)
 	for rep := 0; rep < 2; rep++ {
 		// Push all the data into the stack, pop out every second
 		secs := []*item{}

common/types.go

@@ -76,7 +76,7 @@ func (h Hash) Hex() string { return hexutil.Encode(h[:]) }
 // TerminalString implements log.TerminalStringer, formatting a string for console
 // output during logging.
 func (h Hash) TerminalString() string {
-	return fmt.Sprintf("%x…%x", h[:3], h[29:])
+	return fmt.Sprintf("%x..%x", h[:3], h[29:])
 }
 
 // String implements the stringer interface and is used also by the logger when

consensus/ethash/algorithm.go

@@ -174,7 +174,7 @@ func generateCache(dest []uint32, epoch uint64, seed []byte) {
 			case <-done:
 				return
 			case <-time.After(3 * time.Second):
-				logger.Info("Generating ethash verification cache", "percentage", atomic.LoadUint32(&progress)*100/uint32(rows)/4, "elapsed", common.PrettyDuration(time.Since(start)))
+				logger.Info("Generating ethash verification cache", "percentage", atomic.LoadUint32(&progress)*100/uint32(rows)/(cacheRounds+1), "elapsed", common.PrettyDuration(time.Since(start)))
 			}
 		}
 	}()

consensus/ethash/algorithm_test.go

@@ -726,10 +726,14 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 
 	for i := 0; i < 3; i++ {
 		pend.Add(1)
-
 		go func(idx int) {
 			defer pend.Done()
-			ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
+
+			config := Config{
+				CacheDir:     cachedir,
+				CachesOnDisk: 1,
+			}
+			ethash := New(config, nil, false)
 			defer ethash.Close()
 			if err := ethash.verifySeal(nil, block.Header(), false); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)

consensus/ethash/api.go

@@ -89,7 +89,7 @@ func (api *API) SubmitWork(nonce types.BlockNonce, hash, digest common.Hash) boo
 //
 // It accepts the miner hash rate and an identifier which must be unique
 // between nodes.
-func (api *API) SubmitHashRate(rate hexutil.Uint64, id common.Hash) bool {
+func (api *API) SubmitHashrate(rate hexutil.Uint64, id common.Hash) bool {
 	if api.ethash.remote == nil {
 		return false
 	}

consensus/ethash/consensus.go

@@ -203,15 +203,23 @@ func (ethash *Ethash) VerifyUncles(chain consensus.ChainReader, block *types.Blo
 
 	number, parent := block.NumberU64()-1, block.ParentHash()
 	for i := 0; i < 7; i++ {
+		ancestorHeader := chain.GetHeader(parent, number)
+		if ancestorHeader == nil {
+			break
+		}
+		ancestors[parent] = ancestorHeader
+		// If the ancestor doesn't have any uncles, we don't have to iterate them
+		if ancestorHeader.UncleHash != types.EmptyUncleHash {
+			// Need to add those uncles to the blacklist too
 			ancestor := chain.GetBlock(parent, number)
 			if ancestor == nil {
 				break
 			}
-		ancestors[ancestor.Hash()] = ancestor.Header()
 			for _, uncle := range ancestor.Uncles() {
 				uncles.Add(uncle.Hash())
 			}
-		parent, number = ancestor.ParentHash(), number-1
+		}
+		parent, number = ancestorHeader.ParentHash, number-1
 	}
 	ancestors[block.Hash()] = block.Header()
 	uncles.Add(block.Hash())
@@ -315,6 +323,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uin
 func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int {
 	next := new(big.Int).Add(parent.Number, big1)
 	switch {
+	case config.IsCatalyst(next):
+		return big.NewInt(1)
 	case config.IsMuirGlacier(next):
 		return calcDifficultyEip2384(time, parent)
 	case config.IsConstantinople(next):
@@ -616,6 +626,10 @@ var (
 // reward. The total reward consists of the static block reward and rewards for
 // included uncles. The coinbase of each uncle block is also rewarded.
 func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) {
+	// Skip block reward in catalyst mode
+	if config.IsCatalyst(header.Number) {
+		return
+	}
 	// Select the correct block reward based on chain progression
 	blockReward := FrontierBlockReward
 	if config.IsByzantium(header.Number) {

consensus/ethash/ethash.go

@@ -48,7 +48,7 @@ var (
 	two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false)
+	sharedEthash *Ethash
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -57,6 +57,15 @@ var (
 	dumpMagic = []uint32{0xbaddcafe, 0xfee1dead}
 )
 
+func init() {
+	sharedConfig := Config{
+		PowMode:       ModeNormal,
+		CachesInMem:   3,
+		DatasetsInMem: 1,
+	}
+	sharedEthash = New(sharedConfig, nil, false)
+}
+
 // isLittleEndian returns whether the local system is running in little or big
 // endian byte order.
 func isLittleEndian() bool {
@@ -103,12 +112,13 @@ func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	// Yay, we managed to memory map the file, here be dragons
-	header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem))
-	header.Len /= 4
-	header.Cap /= 4
-
-	return mem, *(*[]uint32)(unsafe.Pointer(&header)), nil
+	// The file is now memory-mapped. Create a []uint32 view of the file.
+	var view []uint32
+	header := (*reflect.SliceHeader)(unsafe.Pointer(&view))
+	header.Data = (*reflect.SliceHeader)(unsafe.Pointer(&mem)).Data
+	header.Cap = len(mem) / 4
+	header.Len = header.Cap
+	return mem, view, nil
 }
 
 // memoryMapAndGenerate tries to memory map a temporary file of uint32s for write
@@ -411,6 +421,10 @@ type Config struct {
 	DatasetsLockMmap bool
 	PowMode          Mode
 
+	// When set, notifications sent by the remote sealer will
+	// be block header JSON objects instead of work package arrays.
+	NotifyFull bool
+
 	Log log.Logger `toml:"-"`
 }
 
@@ -462,6 +476,9 @@ func New(config Config, notify []string, noverify bool) *Ethash {
 		update:   make(chan struct{}),
 		hashrate: metrics.NewMeterForced(),
 	}
+	if config.PowMode == ModeShared {
+		ethash.shared = sharedEthash
+	}
 	ethash.remote = startRemoteSealer(ethash, notify, noverify)
 	return ethash
 }
@@ -469,15 +486,7 @@ func New(config Config, notify []string, noverify bool) *Ethash {
 // NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
 func NewTester(notify []string, noverify bool) *Ethash {
-	ethash := &Ethash{
-		config:   Config{PowMode: ModeTest, Log: log.Root()},
-		caches:   newlru("cache", 1, newCache),
-		datasets: newlru("dataset", 1, newDataset),
-		update:   make(chan struct{}),
-		hashrate: metrics.NewMeterForced(),
-	}
-	ethash.remote = startRemoteSealer(ethash, notify, noverify)
-	return ethash
+	return New(Config{PowMode: ModeTest}, notify, noverify)
 }
 
 // NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts
@@ -537,7 +546,6 @@ func NewShared() *Ethash {
 
 // Close closes the exit channel to notify all backend threads exiting.
 func (ethash *Ethash) Close() error {
-	var err error
 	ethash.closeOnce.Do(func() {
 		// Short circuit if the exit channel is not allocated.
 		if ethash.remote == nil {
@@ -546,7 +554,7 @@ func (ethash *Ethash) Close() error {
 		close(ethash.remote.requestExit)
 		<-ethash.remote.exitCh
 	})
-	return err
+	return nil
 }
 
 // cache tries to retrieve a verification cache for the specified block number

consensus/ethash/ethash_test.go

@@ -62,7 +62,14 @@ func TestCacheFileEvict(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpdir)
-	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false)
+
+	config := Config{
+		CachesInMem:  3,
+		CachesOnDisk: 10,
+		CacheDir:     tmpdir,
+		PowMode:      ModeTest,
+	}
+	e := New(config, nil, false)
 	defer e.Close()
 
 	workers := 8
@@ -128,7 +135,7 @@ func TestRemoteSealer(t *testing.T) {
 	}
 }
 
-func TestHashRate(t *testing.T) {
+func TestHashrate(t *testing.T) {
 	var (
 		hashrate = []hexutil.Uint64{100, 200, 300}
 		expect   uint64
@@ -143,7 +150,7 @@ func TestHashRate(t *testing.T) {
 
 	api := &API{ethash}
 	for i := 0; i < len(hashrate); i += 1 {
-		if res := api.SubmitHashRate(hashrate[i], ids[i]); !res {
+		if res := api.SubmitHashrate(hashrate[i], ids[i]); !res {
 			t.Error("remote miner submit hashrate failed")
 		}
 		expect += uint64(hashrate[i])
@@ -163,7 +170,7 @@ func TestClosedRemoteSealer(t *testing.T) {
 		t.Error("expect to return an error to indicate ethash is stopped")
 	}
 
-	if res := api.SubmitHashRate(hexutil.Uint64(100), common.HexToHash("a")); res {
+	if res := api.SubmitHashrate(hexutil.Uint64(100), common.HexToHash("a")); res {
 		t.Error("expect to return false when submit hashrate to a stopped ethash")
 	}
 }

consensus/ethash/sealer.go

@@ -358,7 +358,16 @@ func (s *remoteSealer) makeWork(block *types.Block) {
 // new work to be processed.
 func (s *remoteSealer) notifyWork() {
 	work := s.currentWork
-	blob, _ := json.Marshal(work)
+
+	// Encode the JSON payload of the notification. When NotifyFull is set,
+	// this is the complete block header, otherwise it is a JSON array.
+	var blob []byte
+	if s.ethash.config.NotifyFull {
+		blob, _ = json.Marshal(s.currentBlock.Header())
+	} else {
+		blob, _ = json.Marshal(work)
+	}
+
 	s.reqWG.Add(len(s.notifyURLs))
 	for _, url := range s.notifyURLs {
 		go s.sendNotification(s.notifyCtx, url, blob, work)

consensus/ethash/sealer_test.go

@@ -22,6 +22,7 @@ import (
 	"math/big"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 	"time"
 
@@ -74,6 +75,50 @@ func TestRemoteNotify(t *testing.T) {
 	}
 }
 
+// Tests whether remote HTTP servers are correctly notified of new work. (Full pending block body / --miner.notify.full)
+func TestRemoteNotifyFull(t *testing.T) {
+	// Start a simple web server to capture notifications.
+	sink := make(chan map[string]interface{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		blob, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			t.Errorf("failed to read miner notification: %v", err)
+		}
+		var work map[string]interface{}
+		if err := json.Unmarshal(blob, &work); err != nil {
+			t.Errorf("failed to unmarshal miner notification: %v", err)
+		}
+		sink <- work
+	}))
+	defer server.Close()
+
+	// Create the custom ethash engine.
+	config := Config{
+		PowMode:    ModeTest,
+		NotifyFull: true,
+		Log:        testlog.Logger(t, log.LvlWarn),
+	}
+	ethash := New(config, []string{server.URL}, false)
+	defer ethash.Close()
+
+	// Stream a work task and ensure the notification bubbles out.
+	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
+	block := types.NewBlockWithHeader(header)
+
+	ethash.Seal(nil, block, nil, nil)
+	select {
+	case work := <-sink:
+		if want := "0x" + strconv.FormatUint(header.Number.Uint64(), 16); work["number"] != want {
+			t.Errorf("pending block number mismatch: have %v, want %v", work["number"], want)
+		}
+		if want := "0x" + header.Difficulty.Text(16); work["difficulty"] != want {
+			t.Errorf("pending block difficulty mismatch: have %s, want %s", work["difficulty"], want)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatalf("notification timed out")
+	}
+}
+
 // Tests that pushing work packages fast to the miner doesn't cause any data race
 // issues in the notifications.
 func TestRemoteMultiNotify(t *testing.T) {
@@ -119,6 +164,55 @@ func TestRemoteMultiNotify(t *testing.T) {
 	}
 }
 
+// Tests that pushing work packages fast to the miner doesn't cause any data race
+// issues in the notifications. Full pending block body / --miner.notify.full)
+func TestRemoteMultiNotifyFull(t *testing.T) {
+	// Start a simple web server to capture notifications.
+	sink := make(chan map[string]interface{}, 64)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		blob, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			t.Errorf("failed to read miner notification: %v", err)
+		}
+		var work map[string]interface{}
+		if err := json.Unmarshal(blob, &work); err != nil {
+			t.Errorf("failed to unmarshal miner notification: %v", err)
+		}
+		sink <- work
+	}))
+	defer server.Close()
+
+	// Create the custom ethash engine.
+	config := Config{
+		PowMode:    ModeTest,
+		NotifyFull: true,
+		Log:        testlog.Logger(t, log.LvlWarn),
+	}
+	ethash := New(config, []string{server.URL}, false)
+	defer ethash.Close()
+
+	// Provide a results reader.
+	// Otherwise the unread results will be logged asynchronously
+	// and this can happen after the test is finished, causing a panic.
+	results := make(chan *types.Block, cap(sink))
+
+	// Stream a lot of work task and ensure all the notifications bubble out.
+	for i := 0; i < cap(sink); i++ {
+		header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)}
+		block := types.NewBlockWithHeader(header)
+		ethash.Seal(nil, block, results, nil)
+	}
+
+	for i := 0; i < cap(sink); i++ {
+		select {
+		case <-sink:
+			<-results
+		case <-time.After(10 * time.Second):
+			t.Fatalf("notification %d timed out", i)
+		}
+	}
+}
+
 // Tests whether stale solutions are correctly processed.
 func TestStaleSubmission(t *testing.T) {
 	ethash := NewTester(nil, true)

core/bench_test.go

@@ -157,7 +157,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 			b.Fatalf("cannot create temporary directory: %v", err)
 		}
 		defer os.RemoveAll(dir)
-		db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "")
+		db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
 		if err != nil {
 			b.Fatalf("cannot create temporary database: %v", err)
 		}
@@ -255,7 +255,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("cannot create temporary directory: %v", err)
 		}
-		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
+		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
@@ -272,7 +272,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 	}
 	defer os.RemoveAll(dir)
 
-	db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
+	db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
 	if err != nil {
 		b.Fatalf("error opening database at %v: %v", dir, err)
 	}
@@ -283,7 +283,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "")
+		db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}

core/blockchain.go

@@ -209,7 +209,6 @@ type BlockChain struct {
 
 	shouldPreserve  func(*types.Block) bool        // Function used to determine whether should preserve the given block.
 	terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion.
-	writeLegacyJournal bool                           // Testing flag used to flush the snapshot journal in legacy format.
 
 	// Bor related changes
 	borReceiptsCache *lru.Cache             // Cache for the most recent bor receipt receipts per block
@@ -492,7 +491,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 
 // SetHeadBeyondRoot rewinds the local chain to a new head with the extra condition
 // that the rewind must pass the specified state root. This method is meant to be
-// used when rewiding with snapshots enabled to ensure that we go back further than
+// used when rewinding with snapshots enabled to ensure that we go back further than
 // persistent disk layer. Depending on whether the node was fast synced or full, and
 // in which state, the method will try to delete minimal data from disk whilst
 // retaining chain consistency.
@@ -642,7 +641,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	// Make sure that both the block as well at its state trie exists
 	block := bc.GetBlockByHash(hash)
 	if block == nil {
-		return fmt.Errorf("non existent block [%x…]", hash[:4])
+		return fmt.Errorf("non existent block [%x..]", hash[:4])
 	}
 	if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil {
 		return err
@@ -653,7 +652,8 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
 	headBlockGauge.Update(int64(block.NumberU64()))
 	bc.chainmu.Unlock()
 
-	// Destroy any existing state snapshot and regenerate it in the background
+	// Destroy any existing state snapshot and regenerate it in the background,
+	// also resuming the normal maintenance of any previously paused snapshot.
 	if bc.snaps != nil {
 		bc.snaps.Rebuild(block.Root())
 	}
@@ -1014,16 +1014,10 @@ func (bc *BlockChain) Stop() {
 	var snapBase common.Hash
 	if bc.snaps != nil {
 		var err error
-		if bc.writeLegacyJournal {
-			if snapBase, err = bc.snaps.LegacyJournal(bc.CurrentBlock().Root()); err != nil {
-				log.Error("Failed to journal state snapshot", "err", err)
-			}
-		} else {
 		if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root()); err != nil {
 			log.Error("Failed to journal state snapshot", "err", err)
 		}
 	}
-	}
 	// Ensure the state of a recent block is also stored to disk before exiting.
 	// We're writing three different states to catch different restart scenarios:
 	//  - HEAD:     So we don't need to reprocess any blocks in the general case
@@ -1159,7 +1153,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() {
 				log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(),
 					"prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash())
-				return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(),
+				return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, blockChain[i-1].NumberU64(),
 					blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4])
 			}
 		}
@@ -1224,65 +1218,16 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			}
 			// Short circuit if the owner header is unknown
 			if !bc.HasHeader(block.Hash(), block.NumberU64()) {
-				return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
+				return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4])
 			}
-			var (
-				start  = time.Now()
-				logged = time.Now()
-				count  int
-			)
-			// Migrate all ancient blocks. This can happen if someone upgrades from Geth
-			// 1.8.x to 1.9.x mid-fast-sync. Perhaps we can get rid of this path in the
-			// long term.
-			for {
-				// We can ignore the error here since light client won't hit this code path.
-				frozen, _ := bc.db.Ancients()
-				if frozen >= block.NumberU64() {
-					break
+			if block.NumberU64() == 1 {
+				// Make sure to write the genesis into the freezer
+				if frozen, _ := bc.db.Ancients(); frozen == 0 {
+					h := rawdb.ReadCanonicalHash(bc.db, 0)
+					b := rawdb.ReadBlock(bc.db, h, 0)
+					size += rawdb.WriteAncientBlock(bc.db, b, rawdb.ReadReceipts(bc.db, h, 0, bc.chainConfig), rawdb.ReadTd(bc.db, h, 0), rawdb.ReadBorReceipt(bc.db, h, frozen))
+					log.Info("Wrote genesis to ancients")
 				}
-				h := rawdb.ReadCanonicalHash(bc.db, frozen)
-				b := rawdb.ReadBlock(bc.db, h, frozen)
-				size += rawdb.WriteAncientBlock(bc.db, b, rawdb.ReadReceipts(bc.db, h, frozen, bc.chainConfig), rawdb.ReadTd(bc.db, h, frozen), rawdb.ReadBorReceipt(bc.db, h, frozen))
-				count += 1
-
-				// Always keep genesis block in active database.
-				if b.NumberU64() != 0 {
-					deleted = append(deleted, &numberHash{b.NumberU64(), b.Hash()})
-				}
-				if time.Since(logged) > 8*time.Second {
-					log.Info("Migrating ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
-					logged = time.Now()
-				}
-				// Don't collect too much in-memory, write it out every 100K blocks
-				if len(deleted) > 100000 {
-					// Sync the ancient store explicitly to ensure all data has been flushed to disk.
-					if err := bc.db.Sync(); err != nil {
-						return 0, err
-					}
-					// Wipe out canonical block data.
-					for _, nh := range deleted {
-						rawdb.DeleteBlockWithoutNumber(batch, nh.hash, nh.number)
-						rawdb.DeleteCanonicalHash(batch, nh.number)
-					}
-					if err := batch.Write(); err != nil {
-						return 0, err
-					}
-					batch.Reset()
-					// Wipe out side chain too.
-					for _, nh := range deleted {
-						for _, hash := range rawdb.ReadAllHashes(bc.db, nh.number) {
-							rawdb.DeleteBlock(batch, hash, nh.number)
-						}
-					}
-					if err := batch.Write(); err != nil {
-						return 0, err
-					}
-					batch.Reset()
-					deleted = deleted[0:]
-				}
-			}
-			if count > 0 {
-				log.Info("Migrated ancient blocks", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
 			}
 			// Flush data into ancient database.
 			size += rawdb.WriteAncientBlock(bc.db, block, receiptChain[i], bc.GetTd(block.Hash(), block.NumberU64()), bc.GetBorReceiptByHash(block.Hash()))
@@ -1368,7 +1313,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			}
 			// Short circuit if the owner header is unknown
 			if !bc.HasHeader(block.Hash(), block.NumberU64()) {
-				return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
+				return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4])
 			}
 			if !skipPresenceCheck {
 				// Ignore if the entire data is already known
@@ -1736,7 +1681,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 			log.Error("Non contiguous block insert", "number", block.Number(), "hash", block.Hash(),
 				"parent", block.ParentHash(), "prevnumber", prev.Number(), "prevhash", prev.Hash())
 
-			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, prev.NumberU64(),
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, prev.NumberU64(),
 				prev.Hash().Bytes()[:4], i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
 		}
 	}
@@ -1750,6 +1695,22 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
 	return n, err
 }
 
+// InsertChainWithoutSealVerification works exactly the same
+// except for seal verification, seal verification is omitted
+func (bc *BlockChain) InsertChainWithoutSealVerification(block *types.Block) (int, error) {
+	bc.blockProcFeed.Send(true)
+	defer bc.blockProcFeed.Send(false)
+
+	// Pre-checks passed, start the full block imports
+	bc.wg.Add(1)
+	bc.chainmu.Lock()
+	n, err := bc.insertChain(types.Blocks([]*types.Block{block}), false)
+	bc.chainmu.Unlock()
+	bc.wg.Done()
+
+	return n, err
+}
+
 // insertChain is the internal implementation of InsertChain, which assumes that
 // 1) chains are contiguous, and 2) The chain mutex is held.
 //
@@ -2222,7 +2183,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
 					l := *log
 					if removed {
 						l.Removed = true
-					} else {
 					}
 					logs = append(logs, &l)
 				}

core/blockchain_repair_test.go

@@ -1762,7 +1762,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 	os.RemoveAll(datadir)
 
-	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
+	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
 	}
@@ -1817,7 +1817,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze(threshold uint64)
+		Freeze(threshold uint64) error
 		Ancients() (uint64, error)
 	}
 	db.(freezer).Freeze(tt.freezeThreshold)
@@ -1830,7 +1830,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	db.Close()
 
 	// Start a new blockchain back up and see where the repait leads us
-	db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
+	db, err = rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
 	}

core/blockchain_sethead_test.go

@@ -1961,7 +1961,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 	os.RemoveAll(datadir)
 
-	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
+	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
 	}
@@ -2023,7 +2023,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze(threshold uint64)
+		Freeze(threshold uint64) error
 		Ancients() (uint64, error)
 	}
 	db.(freezer).Freeze(tt.freezeThreshold)

core/blockchain_snapshot_test.go

@@ -39,7 +39,6 @@ import (
 
 // snapshotTestBasic wraps the common testing fields in the snapshot tests.
 type snapshotTestBasic struct {
-	legacy        bool   // Wether write the snapshot journal in legacy format
 	chainBlocks   int    // Number of blocks to generate for the canonical chain
 	snapshotBlock uint64 // Block number of the relevant snapshot disk layer
 	commitBlock   uint64 // Block number for which to commit the state to disk
@@ -65,7 +64,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 	}
 	os.RemoveAll(datadir)
 
-	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "")
+	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
 	}
@@ -104,11 +103,6 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 			chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil)
 		}
 		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
-			if basic.legacy {
-				// Here we commit the snapshot disk root to simulate
-				// committing the legacy snapshot.
-				rawdb.WriteSnapshotRoot(db, blocks[point-1].Root())
-			} else {
 			// Flushing the entire snap tree into the disk, the
 			// relavant (a) snapshot root and (b) snapshot generator
 			// will be persisted atomically.
@@ -119,7 +113,6 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 			}
 		}
 	}
-	}
 	if _, err := chain.InsertChain(blocks[startPoint:]); err != nil {
 		t.Fatalf("Failed to import canonical chain tail: %v", err)
 	}
@@ -129,12 +122,6 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 	basic.db = db
 	basic.gendb = gendb
 	basic.engine = engine
-
-	// Ugly hack, notify the chain to flush the journal in legacy format
-	// if it's requested.
-	if basic.legacy {
-		chain.writeLegacyJournal = true
-	}
 	return chain, blocks
 }
 
@@ -261,7 +248,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	db.Close()
 
 	// Start a new blockchain back up and see where the repair leads us
-	newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "")
+	newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false)
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
 	}
@@ -484,46 +471,6 @@ func TestRestartWithNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : G
 	test := &snapshotTest{
 		snapshotTestBasic{
-			legacy:             false,
-			chainBlocks:        8,
-			snapshotBlock:      0,
-			commitBlock:        0,
-			expCanonicalBlocks: 8,
-			expHeadHeader:      8,
-			expHeadFastBlock:   8,
-			expHeadBlock:       8,
-			expSnapshotBottom:  0, // Initial disk layer built from genesis
-		},
-	}
-	test.test(t)
-	test.teardown()
-}
-
-// Tests a Geth restart with valid but "legacy" snapshot. Before the shutdown,
-// all snapshot journal will be persisted correctly. In this case no snapshot
-// recovery is required.
-func TestRestartWithLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G
-	// Snapshot: G
-	//
-	// SetHead(0)
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8
-	//
-	// Expected head header    : C8
-	// Expected head fast block: C8
-	// Expected head block     : C8
-	// Expected snapshot disk  : G
-	t.Skip("Legacy format testing is not supported")
-	test := &snapshotTest{
-		snapshotTestBasic{
-			legacy:             true,
 			chainBlocks:        8,
 			snapshotBlock:      0,
 			commitBlock:        0,
@@ -563,7 +510,6 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : C4
 	test := &crashSnapshotTest{
 		snapshotTestBasic{
-			legacy:             false,
 			chainBlocks:        8,
 			snapshotBlock:      4,
 			commitBlock:        0,
@@ -603,7 +549,6 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : C4
 	test := &crashSnapshotTest{
 		snapshotTestBasic{
-			legacy:             false,
 			chainBlocks:        8,
 			snapshotBlock:      4,
 			commitBlock:        2,
@@ -643,7 +588,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : C4
 	test := &crashSnapshotTest{
 		snapshotTestBasic{
-			legacy:             false,
 			chainBlocks:        8,
 			snapshotBlock:      4,
 			commitBlock:        6,
@@ -658,131 +602,6 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
 	test.teardown()
 }
 
-// Tests a Geth was crashed and restarts with a broken and "legacy format"
-// snapshot. In this case the entire legacy snapshot should be discared
-// and rebuild from the new chain head. The new head here refers to the
-// genesis because there is no committed point.
-func TestNoCommitCrashWithLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G
-	// Snapshot: G, C4
-	//
-	// CRASH
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8
-	//
-	// Expected head header    : C8
-	// Expected head fast block: C8
-	// Expected head block     : G
-	// Expected snapshot disk  : G
-	t.Skip("Legacy format testing is not supported")
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			legacy:             true,
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			commitBlock:        0,
-			expCanonicalBlocks: 8,
-			expHeadHeader:      8,
-			expHeadFastBlock:   8,
-			expHeadBlock:       0,
-			expSnapshotBottom:  0, // Rebuilt snapshot from the latest HEAD(genesis)
-		},
-	}
-	test.test(t)
-	test.teardown()
-}
-
-// Tests a Geth was crashed and restarts with a broken and "legacy format"
-// snapshot. In this case the entire legacy snapshot should be discared
-// and rebuild from the new chain head. The new head here refers to the
-// block-2 because it's committed into the disk.
-func TestLowCommitCrashWithLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G, C2
-	// Snapshot: G, C4
-	//
-	// CRASH
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8
-	//
-	// Expected head header    : C8
-	// Expected head fast block: C8
-	// Expected head block     : C2
-	// Expected snapshot disk  : C2
-	t.Skip("Legacy format testing is not supported")
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			legacy:             true,
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			commitBlock:        2,
-			expCanonicalBlocks: 8,
-			expHeadHeader:      8,
-			expHeadFastBlock:   8,
-			expHeadBlock:       2,
-			expSnapshotBottom:  2, // Rebuilt snapshot from the latest HEAD
-		},
-	}
-	test.test(t)
-	test.teardown()
-}
-
-// Tests a Geth was crashed and restarts with a broken and "legacy format"
-// snapshot. In this case the entire legacy snapshot should be discared
-// and rebuild from the new chain head.
-//
-// The new head here refers to the the genesis, the reason is:
-//   - the state of block-6 is committed into the disk
-//   - the legacy disk layer of block-4 is committed into the disk
-//   - the head is rewound the genesis in order to find an available
-//     state lower than disk layer
-func TestHighCommitCrashWithLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G, C6
-	// Snapshot: G, C4
-	//
-	// CRASH
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8
-	//
-	// Expected head header    : C8
-	// Expected head fast block: C8
-	// Expected head block     : G
-	// Expected snapshot disk  : G
-	t.Skip("Legacy format testing is not supported")
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			legacy:             true,
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			commitBlock:        6,
-			expCanonicalBlocks: 8,
-			expHeadHeader:      8,
-			expHeadFastBlock:   8,
-			expHeadBlock:       0,
-			expSnapshotBottom:  0, // Rebuilt snapshot from the latest HEAD(genesis)
-		},
-	}
-	test.test(t)
-	test.teardown()
-}
-
 // Tests a Geth was running with snapshot enabled. Then restarts without
 // enabling snapshot and after that re-enable the snapshot again. In this
 // case the snapshot should be rebuilt with latest chain head.
@@ -806,47 +625,6 @@ func TestGappedNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : C10
 	test := &gappedSnapshotTest{
 		snapshotTestBasic: snapshotTestBasic{
-			legacy:             false,
-			chainBlocks:        8,
-			snapshotBlock:      0,
-			commitBlock:        0,
-			expCanonicalBlocks: 10,
-			expHeadHeader:      10,
-			expHeadFastBlock:   10,
-			expHeadBlock:       10,
-			expSnapshotBottom:  10, // Rebuilt snapshot from the latest HEAD
-		},
-		gapped: 2,
-	}
-	test.test(t)
-	test.teardown()
-}
-
-// Tests a Geth was running with leagcy snapshot enabled. Then restarts
-// without enabling snapshot and after that re-enable the snapshot again.
-// In this case the snapshot should be rebuilt with latest chain head.
-func TestGappedLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G
-	// Snapshot: G
-	//
-	// SetHead(0)
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
-	//
-	// Expected head header    : C10
-	// Expected head fast block: C10
-	// Expected head block     : C10
-	// Expected snapshot disk  : C10
-	t.Skip("Legacy format testing is not supported")
-	test := &gappedSnapshotTest{
-		snapshotTestBasic: snapshotTestBasic{
-			legacy:             true,
 			chainBlocks:        8,
 			snapshotBlock:      0,
 			commitBlock:        0,
@@ -885,7 +663,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
 	// Expected snapshot disk  : G
 	test := &setHeadSnapshotTest{
 		snapshotTestBasic: snapshotTestBasic{
-			legacy:             false,
 			chainBlocks:        8,
 			snapshotBlock:      0,
 			commitBlock:        0,
@@ -901,88 +678,6 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
 	test.teardown()
 }
 
-// Tests the Geth was running with snapshot(legacy-format) enabled and resetHead
-// is applied. In this case the head is rewound to the target(with state available).
-// After that the chain is restarted and the original disk layer is kept.
-func TestSetHeadWithLegacySnapshot(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G
-	// Snapshot: G
-	//
-	// SetHead(4)
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4
-	//
-	// Expected head header    : C4
-	// Expected head fast block: C4
-	// Expected head block     : C4
-	// Expected snapshot disk  : G
-	t.Skip("Legacy format testing is not supported")
-	test := &setHeadSnapshotTest{
-		snapshotTestBasic: snapshotTestBasic{
-			legacy:             true,
-			chainBlocks:        8,
-			snapshotBlock:      0,
-			commitBlock:        0,
-			expCanonicalBlocks: 4,
-			expHeadHeader:      4,
-			expHeadFastBlock:   4,
-			expHeadBlock:       4,
-			expSnapshotBottom:  0, // The initial disk layer is built from the genesis
-		},
-		setHead: 4,
-	}
-	test.test(t)
-	test.teardown()
-}
-
-// Tests the Geth was running with snapshot(legacy-format) enabled and upgrades
-// the disk layer journal(journal generator) to latest format. After that the Geth
-// is restarted from a crash. In this case Geth will find the new-format disk layer
-// journal but with legacy-format diff journal(the new-format is never committed),
-// and the invalid diff journal is expected to be dropped.
-func TestRecoverSnapshotFromCrashWithLegacyDiffJournal(t *testing.T) {
-	// Chain:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
-	//
-	// Commit:   G
-	// Snapshot: G
-	//
-	// SetHead(0)
-	//
-	// ------------------------------
-	//
-	// Expected in leveldb:
-	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
-	//
-	// Expected head header    : C10
-	// Expected head fast block: C10
-	// Expected head block     : C8
-	// Expected snapshot disk  : C10
-	t.Skip("Legacy format testing is not supported")
-	test := &restartCrashSnapshotTest{
-		snapshotTestBasic: snapshotTestBasic{
-			legacy:             true,
-			chainBlocks:        8,
-			snapshotBlock:      0,
-			commitBlock:        0,
-			expCanonicalBlocks: 10,
-			expHeadHeader:      10,
-			expHeadFastBlock:   10,
-			expHeadBlock:       8,  // The persisted state in the first running
-			expSnapshotBottom:  10, // The persisted disk layer in the second running
-		},
-		newBlocks: 2,
-	}
-	test.test(t)
-	test.teardown()
-}
-
 // Tests the Geth was running with a complete snapshot and then imports a few
 // more new blocks on top without enabling the snapshot. After the restart,
 // crash happens. Check everything is ok after the restart.
@@ -1006,7 +701,6 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
 	// Expected snapshot disk  : C10
 	test := &wipeCrashSnapshotTest{
 		snapshotTestBasic: snapshotTestBasic{
-			legacy:             false,
 			chainBlocks:        8,
 			snapshotBlock:      4,
 			commitBlock:        0,

core/blockchain_test.go

@@ -651,7 +651,7 @@ func TestFastVsFullChains(t *testing.T) {
 		t.Fatalf("failed to create temp freezer dir: %v", err)
 	}
 	defer os.Remove(frdir)
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -725,7 +725,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 			t.Fatalf("failed to create temp freezer dir: %v", err)
 		}
 		defer os.Remove(dir)
-		db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "")
+		db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false)
 		if err != nil {
 			t.Fatalf("failed to create temp freezer db: %v", err)
 		}
@@ -1462,13 +1462,13 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
 			t.Fatalf("block %d: failed to insert into chain: %v", i, err)
 		}
 		if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() {
-			t.Errorf("block %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
+			t.Errorf("block %d: current block/header mismatch: block #%d [%x..], header #%d [%x..]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
 		}
 		if _, err := chain.InsertChain(forks[i : i+1]); err != nil {
 			t.Fatalf(" fork %d: failed to insert into chain: %v", i, err)
 		}
 		if chain.CurrentBlock().Hash() != chain.CurrentHeader().Hash() {
-			t.Errorf(" fork %d: current block/header mismatch: block #%d [%x…], header #%d [%x…]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
+			t.Errorf(" fork %d: current block/header mismatch: block #%d [%x..], header #%d [%x..]", i, chain.CurrentBlock().Number(), chain.CurrentBlock().Hash().Bytes()[:4], chain.CurrentHeader().Number, chain.CurrentHeader().Hash().Bytes()[:4])
 		}
 	}
 }
@@ -1592,7 +1592,7 @@ func TestBlockchainRecovery(t *testing.T) {
 	}
 	defer os.Remove(frdir)
 
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -1649,7 +1649,7 @@ func TestIncompleteAncientReceiptChainInsertion(t *testing.T) {
 		t.Fatalf("failed to create temp freezer dir: %v", err)
 	}
 	defer os.Remove(frdir)
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -1848,7 +1848,7 @@ func testInsertKnownChainData(t *testing.T, typ string) {
 		t.Fatalf("failed to create temp freezer dir: %v", err)
 	}
 	defer os.Remove(dir)
-	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "")
+	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2128,7 +2128,7 @@ func TestTransactionIndices(t *testing.T) {
 		t.Fatalf("failed to create temp freezer dir: %v", err)
 	}
 	defer os.Remove(frdir)
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2156,7 +2156,7 @@ func TestTransactionIndices(t *testing.T) {
 	// Init block chain with external ancients, check all needed indices has been indexed.
 	limit := []uint64{0, 32, 64, 128}
 	for _, l := range limit {
-		ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+		ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 		if err != nil {
 			t.Fatalf("failed to create temp freezer db: %v", err)
 		}
@@ -2176,7 +2176,7 @@ func TestTransactionIndices(t *testing.T) {
 	}
 
 	// Reconstruct a block chain which only reserves HEAD-64 tx indices
-	ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2255,7 +2255,7 @@ func TestSkipStaleTxIndicesInFastSync(t *testing.T) {
 		t.Fatalf("failed to create temp freezer dir: %v", err)
 	}
 	defer os.Remove(frdir)
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "")
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}

core/chain_indexer.go

@@ -401,7 +401,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com
 		}
 		header := rawdb.ReadHeader(c.chainDb, hash, number)
 		if header == nil {
-			return common.Hash{}, fmt.Errorf("block #%d [%x…] not found", number, hash[:4])
+			return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4])
 		} else if header.ParentHash != lastHead {
 			return common.Hash{}, fmt.Errorf("chain reorged during section processing")
 		}

core/chain_indexer_test.go

@@ -18,6 +18,7 @@ package core
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/big"
 	"math/rand"
@@ -224,7 +225,10 @@ func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Heade
 	//t.processCh <- header.Number.Uint64()
 	select {
 	case <-time.After(10 * time.Second):
-		b.t.Fatal("Unexpected call to Process")
+		b.t.Error("Unexpected call to Process")
+		// Can't use Fatal since this is not the test's goroutine.
+		// Returning error stops the chainIndexer's updateLoop
+		return errors.New("Unexpected call to Process")
 	case b.processCh <- header.Number.Uint64():
 	}
 	return nil

core/chain_makers.go

@@ -111,6 +111,11 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) {
 	b.receipts = append(b.receipts, receipt)
 }
 
+// GetBalance returns the balance of the given address at the generated block.
+func (b *BlockGen) GetBalance(addr common.Address) *big.Int {
+	return b.statedb.GetBalance(addr)
+}
+
 // AddUncheckedTx forcefully adds a transaction to the block without any
 // validation.
 //

core/genesis_test.go

@@ -162,3 +162,38 @@ func TestSetupGenesis(t *testing.T) {
 		}
 	}
 }
+
+// TestGenesisHashes checks the congruity of default genesis data to corresponding hardcoded genesis hash values.
+func TestGenesisHashes(t *testing.T) {
+	cases := []struct {
+		genesis *Genesis
+		hash    common.Hash
+	}{
+		{
+			genesis: DefaultGenesisBlock(),
+			hash:    params.MainnetGenesisHash,
+		},
+		{
+			genesis: DefaultGoerliGenesisBlock(),
+			hash:    params.GoerliGenesisHash,
+		},
+		{
+			genesis: DefaultRopstenGenesisBlock(),
+			hash:    params.RopstenGenesisHash,
+		},
+		{
+			genesis: DefaultRinkebyGenesisBlock(),
+			hash:    params.RinkebyGenesisHash,
+		},
+		{
+			genesis: DefaultYoloV3GenesisBlock(),
+			hash:    params.YoloV3GenesisHash,
+		},
+	}
+	for i, c := range cases {
+		b := c.genesis.MustCommit(rawdb.NewMemoryDatabase())
+		if got := b.Hash(); got != c.hash {
+			t.Errorf("case: %d, want: %s, got: %s", i, c.hash.Hex(), got.Hex())
+		}
+	}
+}

core/headerchain.go

@@ -306,7 +306,7 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int)
 			log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", hash,
 				"parent", chain[i].ParentHash, "prevnumber", chain[i-1].Number, "prevhash", parentHash)
 
-			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].Number,
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, chain[i-1].Number,
 				parentHash.Bytes()[:4], i, chain[i].Number, hash.Bytes()[:4], chain[i].ParentHash[:4])
 		}
 		// If the header is a banned one, straight out abort
@@ -323,7 +323,7 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int)
 	seals := make([]bool, len(chain))
 	if checkFreq != 0 {
 		// In case of checkFreq == 0 all seals are left false.
-		for i := 0; i < len(seals)/checkFreq; i++ {
+		for i := 0; i <= len(seals)/checkFreq; i++ {
 			index := i*checkFreq + hc.rand.Intn(checkFreq)
 			if index >= len(seals) {
 				index = len(seals) - 1

core/rawdb/accessors_chain.go

@@ -840,3 +840,29 @@ func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header {
 	}
 	return a
 }
+
+// ReadHeadHeader returns the current canonical head header.
+func ReadHeadHeader(db ethdb.Reader) *types.Header {
+	headHeaderHash := ReadHeadHeaderHash(db)
+	if headHeaderHash == (common.Hash{}) {
+		return nil
+	}
+	headHeaderNumber := ReadHeaderNumber(db, headHeaderHash)
+	if headHeaderNumber == nil {
+		return nil
+	}
+	return ReadHeader(db, headHeaderHash, *headHeaderNumber)
+}
+
+// ReadHeadBlock returns the current canonical head block.
+func ReadHeadBlock(db ethdb.Reader) *types.Block {
+	headBlockHash := ReadHeadBlockHash(db)
+	if headBlockHash == (common.Hash{}) {
+		return nil
+	}
+	headBlockNumber := ReadHeaderNumber(db, headBlockHash)
+	if headBlockNumber == nil {
+		return nil
+	}
+	return ReadBlock(db, headBlockHash, *headBlockNumber)
+}

core/rawdb/accessors_chain_test.go

@@ -440,7 +440,7 @@ func TestAncientStorage(t *testing.T) {
 	}
 	defer os.Remove(frdir)
 
-	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "")
+	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}

core/rawdb/accessors_snapshot.go

@@ -24,6 +24,26 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
+// ReadSnapshotDisabled retrieves if the snapshot maintenance is disabled.
+func ReadSnapshotDisabled(db ethdb.KeyValueReader) bool {
+	disabled, _ := db.Has(snapshotDisabledKey)
+	return disabled
+}
+
+// WriteSnapshotDisabled stores the snapshot pause flag.
+func WriteSnapshotDisabled(db ethdb.KeyValueWriter) {
+	if err := db.Put(snapshotDisabledKey, []byte("42")); err != nil {
+		log.Crit("Failed to store snapshot disabled flag", "err", err)
+	}
+}
+
+// DeleteSnapshotDisabled deletes the flag keeping the snapshot maintenance disabled.
+func DeleteSnapshotDisabled(db ethdb.KeyValueWriter) {
+	if err := db.Delete(snapshotDisabledKey); err != nil {
+		log.Crit("Failed to remove snapshot disabled flag", "err", err)
+	}
+}
+
 // ReadSnapshotRoot retrieves the root of the block whose state is contained in
 // the persisted snapshot.
 func ReadSnapshotRoot(db ethdb.KeyValueReader) common.Hash {

core/rawdb/chain_iterator.go

@@ -23,10 +23,10 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/prque"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
-	"golang.org/x/crypto/sha3"
 )
 
 // InitDatabaseFromFreezer reinitializes an empty database from a previous batch
@@ -135,32 +135,15 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
 				close(hashesCh)
 			}
 		}()
-
-		var hasher = sha3.NewLegacyKeccak256()
 		for data := range rlpCh {
-			it, err := rlp.NewListIterator(data.rlp)
-			if err != nil {
-				log.Warn("tx iteration error", "error", err)
-				return
-			}
-			it.Next()
-			txs := it.Value()
-			txIt, err := rlp.NewListIterator(txs)
-			if err != nil {
-				log.Warn("tx iteration error", "error", err)
+			var body types.Body
+			if err := rlp.DecodeBytes(data.rlp, &body); err != nil {
+				log.Warn("Failed to decode block body", "block", data.number, "error", err)
 				return
 			}
 			var hashes []common.Hash
-			for txIt.Next() {
-				if err := txIt.Err(); err != nil {
-					log.Warn("tx iteration error", "error", err)
-					return
-				}
-				var txHash common.Hash
-				hasher.Reset()
-				hasher.Write(txIt.Value())
-				hasher.Sum(txHash[:0])
-				hashes = append(hashes, txHash)
+			for _, tx := range body.Transactions {
+				hashes = append(hashes, tx.Hash())
 			}
 			result := &blockTxHashes{
 				hashes: hashes,

core/rawdb/chain_iterator_test.go

@@ -33,14 +33,34 @@ func TestChainIterator(t *testing.T) {
 
 	var block *types.Block
 	var txs []*types.Transaction
-	for i := uint64(0); i <= 10; i++ {
-		if i == 0 {
-			block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil, newHasher()) // Empty genesis block
+	to := common.BytesToAddress([]byte{0x11})
+	block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, nil, newHasher()) // Empty genesis block
+	WriteBlock(chainDb, block)
+	WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
+	for i := uint64(1); i <= 10; i++ {
+		var tx *types.Transaction
+		if i%2 == 0 {
+			tx = types.NewTx(&types.LegacyTx{
+				Nonce:    i,
+				GasPrice: big.NewInt(11111),
+				Gas:      1111,
+				To:       &to,
+				Value:    big.NewInt(111),
+				Data:     []byte{0x11, 0x11, 0x11},
+			})
 		} else {
-			tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+			tx = types.NewTx(&types.AccessListTx{
+				ChainID:  big.NewInt(1337),
+				Nonce:    i,
+				GasPrice: big.NewInt(11111),
+				Gas:      1111,
+				To:       &to,
+				Value:    big.NewInt(111),
+				Data:     []byte{0x11, 0x11, 0x11},
+			})
+		}
 		txs = append(txs, tx)
 		block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher())
-		}
 		WriteBlock(chainDb, block)
 		WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
 	}
@@ -66,7 +86,7 @@ func TestChainIterator(t *testing.T) {
 				numbers = append(numbers, int(h.number))
 				if len(h.hashes) > 0 {
 					if got, exp := h.hashes[0], txs[h.number-1].Hash(); got != exp {
-						t.Fatalf("hash wrong, got %x exp %x", got, exp)
+						t.Fatalf("block %d: hash wrong, got %x exp %x", h.number, got, exp)
 					}
 				}
 			}
@@ -88,14 +108,37 @@ func TestIndexTransactions(t *testing.T) {
 
 	var block *types.Block
 	var txs []*types.Transaction
-	for i := uint64(0); i <= 10; i++ {
-		if i == 0 {
-			block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, nil, nil, nil, newHasher()) // Empty genesis block
+	to := common.BytesToAddress([]byte{0x11})
+
+	// Write empty genesis block
+	block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, nil, newHasher())
+	WriteBlock(chainDb, block)
+	WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
+
+	for i := uint64(1); i <= 10; i++ {
+		var tx *types.Transaction
+		if i%2 == 0 {
+			tx = types.NewTx(&types.LegacyTx{
+				Nonce:    i,
+				GasPrice: big.NewInt(11111),
+				Gas:      1111,
+				To:       &to,
+				Value:    big.NewInt(111),
+				Data:     []byte{0x11, 0x11, 0x11},
+			})
 		} else {
-			tx := types.NewTransaction(i, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+			tx = types.NewTx(&types.AccessListTx{
+				ChainID:  big.NewInt(1337),
+				Nonce:    i,
+				GasPrice: big.NewInt(11111),
+				Gas:      1111,
+				To:       &to,
+				Value:    big.NewInt(111),
+				Data:     []byte{0x11, 0x11, 0x11},
+			})
+		}
 		txs = append(txs, tx)
 		block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, []*types.Transaction{tx}, nil, nil, newHasher())
-		}
 		WriteBlock(chainDb, block)
 		WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
 	}
@@ -108,10 +151,10 @@ func TestIndexTransactions(t *testing.T) {
 			}
 			number := ReadTxLookupEntry(chainDb, txs[i-1].Hash())
 			if exist && number == nil {
-				t.Fatalf("Transaction indice missing")
+				t.Fatalf("Transaction index %d missing", i)
 			}
 			if !exist && number != nil {
-				t.Fatalf("Transaction indice is not deleted")
+				t.Fatalf("Transaction index %d is not deleted", i)
 			}
 		}
 		number := ReadTxIndexTail(chainDb)

core/rawdb/database.go

@@ -57,7 +57,10 @@ func (frdb *freezerdb) Close() error {
 // Freeze is a helper method used for external testing to trigger and block until
 // a freeze cycle completes, without having to sleep for a minute to trigger the
 // automatic background run.
-func (frdb *freezerdb) Freeze(threshold uint64) {
+func (frdb *freezerdb) Freeze(threshold uint64) error {
+	if frdb.AncientStore.(*freezer).readonly {
+		return errReadOnly
+	}
 	// Set the freezer threshold to a temporary value
 	defer func(old uint64) {
 		atomic.StoreUint64(&frdb.AncientStore.(*freezer).threshold, old)
@@ -68,6 +71,7 @@ func (frdb *freezerdb) Freeze(threshold uint64) {
 	trigger := make(chan struct{}, 1)
 	frdb.AncientStore.(*freezer).trigger <- trigger
 	<-trigger
+	return nil
 }
 
 // nofreezedb is a database wrapper that disables freezer data retrievals.
@@ -121,9 +125,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
 // NewDatabaseWithFreezer creates a high level database on top of a given key-
 // value data store with a freezer moving immutable chain segments into cold
 // storage.
-func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) {
+func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
 	// Create the idle freezer instance
-	frdb, err := newFreezer(freezer, namespace)
+	frdb, err := newFreezer(freezer, namespace, readonly)
 	if err != nil {
 		return nil, err
 	}
@@ -192,8 +196,9 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 		}
 	}
 	// Freezer is consistent with the key-value database, permit combining the two
+	if !frdb.readonly {
 		go frdb.freeze(db)
-
+	}
 	return &freezerdb{
 		KeyValueStore: db,
 		AncientStore:  frdb,
@@ -215,8 +220,8 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database {
 
 // NewLevelDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
-func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) {
-	db, err := leveldb.New(file, cache, handles, namespace)
+func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+	db, err := leveldb.New(file, cache, handles, namespace, readonly)
 	if err != nil {
 		return nil, err
 	}
@@ -225,12 +230,12 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (
 
 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
 // freezer moving immutable chain segments into cold storage.
-func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) {
-	kvdb, err := leveldb.New(file, cache, handles, namespace)
+func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace)
+	frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly)
 	if err != nil {
 		kvdb.Close()
 		return nil, err
@@ -366,9 +371,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			var accounted bool
 			for _, meta := range [][]byte{
 				databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, lastPivotKey,
-				fastTrieProgressKey, snapshotRootKey, snapshotJournalKey, snapshotGeneratorKey,
-				snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey,
-				badBlockKey,
+				fastTrieProgressKey, snapshotDisabledKey, snapshotRootKey, snapshotJournalKey,
+				snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
+				uncleanShutdownKey, badBlockKey,
 			} {
 				if bytes.Equal(key, meta) {
 					metadata.Add(size)

core/rawdb/database_test.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb

core/rawdb/freezer.go

@@ -35,6 +35,10 @@ import (
 )
 
 var (
+	// errReadOnly is returned if the freezer is opened in read only mode. All the
+	// mutations are disallowed.
+	errReadOnly = errors.New("read only")
+
 	// errUnknownTable is returned if the user attempts to read from a table that is
 	// not tracked by the freezer.
 	errUnknownTable = errors.New("unknown table")
@@ -73,6 +77,7 @@ type freezer struct {
 	frozen    uint64 // Number of blocks already frozen
 	threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
 
+	readonly     bool
 	tables       map[string]*freezerTable // Data tables for storing everything
 	instanceLock fileutil.Releaser        // File-system lock to prevent double opens
 
@@ -84,7 +89,7 @@ type freezer struct {
 
 // newFreezer creates a chain freezer that moves ancient chain data into
 // append-only flat file containers.
-func newFreezer(datadir string, namespace string) (*freezer, error) {
+func newFreezer(datadir string, namespace string, readonly bool) (*freezer, error) {
 	// Create the initial freezer object
 	var (
 		readMeter  = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
@@ -106,13 +111,14 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
 	}
 	// Open all the supported data tables
 	freezer := &freezer{
+		readonly:     readonly,
 		threshold:    params.FullImmutabilityThreshold,
 		tables:       make(map[string]*freezerTable),
 		instanceLock: lock,
 		trigger:      make(chan chan struct{}),
 		quit:         make(chan struct{}),
 	}
-	for name, disableSnappy := range freezerNoSnappy {
+	for name, disableSnappy := range FreezerNoSnappy {
 		table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, disableSnappy)
 		if err != nil {
 			for _, table := range freezer.tables {
@@ -143,7 +149,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
 		lock.Release()
 		return nil, err
 	}
-	log.Info("Opened ancient database", "database", datadir)
+	log.Info("Opened ancient database", "database", datadir, "readonly", readonly)
 	return freezer, nil
 }
 
@@ -151,7 +157,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
 func (f *freezer) Close() error {
 	var errs []error
 	f.closeOnce.Do(func() {
-		f.quit <- struct{}{}
+		close(f.quit)
 		for _, table := range f.tables {
 			if err := table.Close(); err != nil {
 				errs = append(errs, err)
@@ -204,6 +210,9 @@ func (f *freezer) AncientSize(kind string) (uint64, error) {
 // injection will be rejected. But if two injections with same number happen at
 // the same time, we can get into the trouble.
 func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td, borBlockReceipt []byte) (err error) {
+	if f.readonly {
+		return errReadOnly
+	}
 	// Ensure the binary blobs we are appending is continuous with freezer.
 	if atomic.LoadUint64(&f.frozen) != number {
 		return errOutOrderInsertion
@@ -251,6 +260,9 @@ func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td,
 
 // TruncateAncients discards any recent data above the provided threshold number.
 func (f *freezer) TruncateAncients(items uint64) error {
+	if f.readonly {
+		return errReadOnly
+	}
 	if atomic.LoadUint64(&f.frozen) <= items {
 		return nil
 	}

core/rawdb/freezer_table.go

@@ -465,35 +465,59 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) {
 // Note, this method will *not* flush any data to disk so be sure to explicitly
 // fsync before irreversibly deleting data from the database.
 func (t *freezerTable) Append(item uint64, blob []byte) error {
-	// Read lock prevents competition with truncate
-	t.lock.RLock()
-	// Ensure the table is still accessible
-	if t.index == nil || t.head == nil {
-		t.lock.RUnlock()
-		return errClosed
-	}
-	// Ensure only the next item can be written, nothing else
-	if atomic.LoadUint64(&t.items) != item {
-		t.lock.RUnlock()
-		return fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item)
-	}
-	// Encode the blob and write it into the data file
+	// Encode the blob before the lock portion
 	if !t.noCompression {
 		blob = snappy.Encode(nil, blob)
 	}
-	bLen := uint32(len(blob))
+	// Read lock prevents competition with truncate
+	retry, err := t.append(item, blob, false)
+	if err != nil {
+		return err
+	}
+	if retry {
+		// Read lock was insufficient, retry with a writelock
+		_, err = t.append(item, blob, true)
+	}
+	return err
+}
+
+// append injects a binary blob at the end of the freezer table.
+// Normally, inserts do not require holding the write-lock, so it should be invoked with 'wlock' set to
+// false.
+// However, if the data will grown the current file out of bounds, then this
+// method will return 'true, nil', indicating that the caller should retry, this time
+// with 'wlock' set to true.
+func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool, error) {
+	if wlock {
+		t.lock.Lock()
+		defer t.lock.Unlock()
+	} else {
+		t.lock.RLock()
+		defer t.lock.RUnlock()
+	}
+	// Ensure the table is still accessible
+	if t.index == nil || t.head == nil {
+		return false, errClosed
+	}
+	// Ensure only the next item can be written, nothing else
+	if atomic.LoadUint64(&t.items) != item {
+		return false, fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item)
+	}
+	bLen := uint32(len(encodedBlob))
 	if t.headBytes+bLen < bLen ||
 		t.headBytes+bLen > t.maxFileSize {
-		// we need a new file, writing would overflow
-		t.lock.RUnlock()
-		t.lock.Lock()
+		// Writing would overflow, so we need to open a new data file.
+		// If we don't already hold the writelock, abort and let the caller
+		// invoke this method a second time.
+		if !wlock {
+			return true, nil
+		}
 		nextID := atomic.LoadUint32(&t.headId) + 1
 		// We open the next file in truncated mode -- if this file already
 		// exists, we need to start over from scratch on it
 		newHead, err := t.openFile(nextID, openFreezerFileTruncated)
 		if err != nil {
-			t.lock.Unlock()
-			return err
+			return false, err
 		}
 		// Close old file, and reopen in RDONLY mode
 		t.releaseFile(t.headId)
@@ -503,13 +527,9 @@ func (t *freezerTable) Append(item uint64, blob []byte) error {
 		t.head = newHead
 		atomic.StoreUint32(&t.headBytes, 0)
 		atomic.StoreUint32(&t.headId, nextID)
-		t.lock.Unlock()
-		t.lock.RLock()
 	}
-
-	defer t.lock.RUnlock()
-	if _, err := t.head.Write(blob); err != nil {
-		return err
+	if _, err := t.head.Write(encodedBlob); err != nil {
+		return false, err
 	}
 	newOffset := atomic.AddUint32(&t.headBytes, bLen)
 	idx := indexEntry{
@@ -523,7 +543,7 @@ func (t *freezerTable) Append(item uint64, blob []byte) error {
 	t.sizeGauge.Inc(int64(bLen + indexEntrySize))
 
 	atomic.AddUint64(&t.items, 1)
-	return nil
+	return false, nil
 }
 
 // getBounds returns the indexes for the item
@@ -562,45 +582,49 @@ func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) {
 // Retrieve looks up the data offset of an item with the given number and retrieves
 // the raw binary blob from the data file.
 func (t *freezerTable) Retrieve(item uint64) ([]byte, error) {
+	blob, err := t.retrieve(item)
+	if err != nil {
+		return nil, err
+	}
+	if t.noCompression {
+		return blob, nil
+	}
+	return snappy.Decode(nil, blob)
+}
+
+// retrieve looks up the data offset of an item with the given number and retrieves
+// the raw binary blob from the data file. OBS! This method does not decode
+// compressed data.
+func (t *freezerTable) retrieve(item uint64) ([]byte, error) {
 	t.lock.RLock()
+	defer t.lock.RUnlock()
 	// Ensure the table and the item is accessible
 	if t.index == nil || t.head == nil {
-		t.lock.RUnlock()
 		return nil, errClosed
 	}
 	if atomic.LoadUint64(&t.items) <= item {
-		t.lock.RUnlock()
 		return nil, errOutOfBounds
 	}
 	// Ensure the item was not deleted from the tail either
 	if uint64(t.itemOffset) > item {
-		t.lock.RUnlock()
 		return nil, errOutOfBounds
 	}
 	startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset))
 	if err != nil {
-		t.lock.RUnlock()
 		return nil, err
 	}
 	dataFile, exist := t.files[filenum]
 	if !exist {
-		t.lock.RUnlock()
 		return nil, fmt.Errorf("missing data file %d", filenum)
 	}
 	// Retrieve the data itself, decompress and return
 	blob := make([]byte, endOffset-startOffset)
 	if _, err := dataFile.ReadAt(blob, int64(startOffset)); err != nil {
-		t.lock.RUnlock()
 		return nil, err
 	}
-	t.lock.RUnlock()
 	t.readMeter.Mark(int64(len(blob) + 2*indexEntrySize))
-
-	if t.noCompression {
 	return blob, nil
 }
-	return snappy.Decode(nil, blob)
-}
 
 // has returns an indicator whether the specified number data
 // exists in the freezer table.
@@ -636,27 +660,26 @@ func (t *freezerTable) Sync() error {
 	return t.head.Sync()
 }
 
-// printIndex is a debug print utility function for testing
-func (t *freezerTable) printIndex() {
+// DumpIndex is a debug print utility function, mainly for testing. It can also
+// be used to analyse a live freezer table index.
+func (t *freezerTable) DumpIndex(start, stop int64) {
 	buf := make([]byte, indexEntrySize)
 
-	fmt.Printf("|-----------------|\n")
-	fmt.Printf("| fileno | offset |\n")
-	fmt.Printf("|--------+--------|\n")
+	fmt.Printf("| number | fileno | offset |\n")
+	fmt.Printf("|--------|--------|--------|\n")
 
-	for i := uint64(0); ; i++ {
+	for i := uint64(start); ; i++ {
 		if _, err := t.index.ReadAt(buf, int64(i*indexEntrySize)); err != nil {
 			break
 		}
 		var entry indexEntry
 		entry.unmarshalBinary(buf)
-		fmt.Printf("|  %03d   |  %03d   | \n", entry.filenum, entry.offset)
-		if i > 100 {
-			fmt.Printf(" ... \n")
+		fmt.Printf("|  %03d   |  %03d   |  %03d   | \n", i, entry.filenum, entry.offset)
+		if stop > 0 && i >= uint64(stop) {
 			break
 		}
 	}
-	fmt.Printf("|-----------------|\n")
+	fmt.Printf("|--------------------------|\n")
 }
 
 //

core/rawdb/freezer_table_test.go

@@ -18,10 +18,13 @@ package rawdb
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
@@ -525,7 +528,7 @@ func TestOffset(t *testing.T) {
 
 		f.Append(4, getChunk(20, 0xbb))
 		f.Append(5, getChunk(20, 0xaa))
-		f.printIndex()
+		f.DumpIndex(0, 100)
 		f.Close()
 	}
 	// Now crop it.
@@ -572,7 +575,7 @@ func TestOffset(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		f.printIndex()
+		f.DumpIndex(0, 100)
 		// It should allow writing item 6
 		f.Append(numDeleted+2, getChunk(20, 0x99))
 
@@ -637,6 +640,55 @@ func TestOffset(t *testing.T) {
 // 1. have data files d0, d1, d2, d3
 // 2. remove d2,d3
 //
-// However, all 'normal' failure modes arising due to failing to sync() or save a file should be
-// handled already, and the case described above can only (?) happen if an external process/user
-// deletes files from the filesystem.
+// However, all 'normal' failure modes arising due to failing to sync() or save a file
+// should be handled already, and the case described above can only (?) happen if an
+// external process/user deletes files from the filesystem.
+
+// TestAppendTruncateParallel is a test to check if the Append/truncate operations are
+// racy.
+//
+// The reason why it's not a regular fuzzer, within tests/fuzzers, is that it is dependent
+// on timing rather than 'clever' input -- there's no determinism.
+func TestAppendTruncateParallel(t *testing.T) {
+	dir, err := ioutil.TempDir("", "freezer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	f, err := newCustomTable(dir, "tmp", metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, 8, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fill := func(mark uint64) []byte {
+		data := make([]byte, 8)
+		binary.LittleEndian.PutUint64(data, mark)
+		return data
+	}
+
+	for i := 0; i < 5000; i++ {
+		f.truncate(0)
+		data0 := fill(0)
+		f.Append(0, data0)
+		data1 := fill(1)
+
+		var wg sync.WaitGroup
+		wg.Add(2)
+		go func() {
+			f.truncate(0)
+			wg.Done()
+		}()
+		go func() {
+			f.Append(1, data1)
+			wg.Done()
+		}()
+		wg.Wait()
+
+		if have, err := f.Retrieve(0); err == nil {
+			if !bytes.Equal(have, data0) {
+				t.Fatalf("have %x want %x", have, data0)
+			}
+		}
+	}
+}

core/rawdb/schema.go

@@ -45,6 +45,9 @@ var (
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
 	fastTrieProgressKey = []byte("TrieSync")
 
+	// snapshotDisabledKey flags that the snapshot should not be maintained due to initial sync.
+	snapshotDisabledKey = []byte("SnapshotDisabled")
+
 	// snapshotRootKey tracks the hash of the last snapshot.
 	snapshotRootKey = []byte("SnapshotRoot")
 
@@ -114,9 +117,9 @@ const (
 	freezerDifficultyTable = "diffs"
 )
 
-// freezerNoSnappy configures whether compression is disabled for the ancient-tables.
+// FreezerNoSnappy configures whether compression is disabled for the ancient-tables.
 // Hashes and difficulties don't compress well.
-var freezerNoSnappy = map[string]bool{
+var FreezerNoSnappy = map[string]bool{
 	freezerHeaderTable:     false,
 	freezerHashTable:       true,
 	freezerBodiesTable:     false,

core/state/pruner/pruner.go

@@ -85,8 +85,12 @@ type Pruner struct {
 }
 
 // NewPruner creates the pruner instance.
-func NewPruner(db ethdb.Database, headHeader *types.Header, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) {
-	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, false)
+func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) {
+	headBlock := rawdb.ReadHeadBlock(db)
+	if headBlock == nil {
+		return nil, errors.New("Failed to load head block")
+	}
+	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false)
 	if err != nil {
 		return nil, err // The relevant snapshot(s) might not exist
 	}
@@ -104,12 +108,12 @@ func NewPruner(db ethdb.Database, headHeader *types.Header, datadir, trieCachePa
 		stateBloom:    stateBloom,
 		datadir:       datadir,
 		trieCachePath: trieCachePath,
-		headHeader:    headHeader,
+		headHeader:    headBlock.Header(),
 		snaptree:      snaptree,
 	}, nil
 }
 
-func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[common.Hash]struct{}, start time.Time) error {
+func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error {
 	// Delete all stale trie nodes in the disk. With the help of state bloom
 	// the trie nodes(and codes) belong to the active state will be filtered
 	// out. A very small part of stale tries will also be filtered because of
@@ -182,6 +186,25 @@ func prune(maindb ethdb.Database, stateBloom *stateBloom, middleStateRoots map[c
 	iter.Release()
 	log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart)))
 
+	// Pruning is done, now drop the "useless" layers from the snapshot.
+	// Firstly, flushing the target layer into the disk. After that all
+	// diff layers below the target will all be merged into the disk.
+	if err := snaptree.Cap(root, 0); err != nil {
+		return err
+	}
+	// Secondly, flushing the snapshot journal into the disk. All diff
+	// layers upon are dropped silently. Eventually the entire snapshot
+	// tree is converted into a single disk layer with the pruning target
+	// as the root.
+	if _, err := snaptree.Journal(root); err != nil {
+		return err
+	}
+	// Delete the state bloom, it marks the entire pruning procedure is
+	// finished. If any crashes or manual exit happens before this,
+	// `RecoverPruning` will pick it up in the next restarts to redo all
+	// the things.
+	os.RemoveAll(bloomPath)
+
 	// Start compactions, will remove the deleted data from the disk immediately.
 	// Note for small pruning, the compaction is skipped.
 	if count >= rangeCompactionThreshold {
@@ -310,29 +333,7 @@ func (p *Pruner) Prune(root common.Hash) error {
 		return err
 	}
 	log.Info("State bloom filter committed", "name", filterName)
-
-	if err := prune(p.db, p.stateBloom, middleRoots, start); err != nil {
-		return err
-	}
-	// Pruning is done, now drop the "useless" layers from the snapshot.
-	// Firstly, flushing the target layer into the disk. After that all
-	// diff layers below the target will all be merged into the disk.
-	if err := p.snaptree.Cap(root, 0); err != nil {
-		return err
-	}
-	// Secondly, flushing the snapshot journal into the disk. All diff
-	// layers upon the target layer are dropped silently. Eventually the
-	// entire snapshot tree is converted into a single disk layer with
-	// the pruning target as the root.
-	if _, err := p.snaptree.Journal(root); err != nil {
-		return err
-	}
-	// Delete the state bloom, it marks the entire pruning procedure is
-	// finished. If any crashes or manual exit happens before this,
-	// `RecoverPruning` will pick it up in the next restarts to redo all
-	// the things.
-	os.RemoveAll(filterName)
-	return nil
+	return prune(p.snaptree, root, p.db, p.stateBloom, filterName, middleRoots, start)
 }
 
 // RecoverPruning will resume the pruning procedure during the system restart.
@@ -350,9 +351,9 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
 	if stateBloomPath == "" {
 		return nil // nothing to recover
 	}
-	headHeader, err := getHeadHeader(db)
-	if err != nil {
-		return err
+	headBlock := rawdb.ReadHeadBlock(db)
+	if headBlock == nil {
+		return errors.New("Failed to load head block")
 	}
 	// Initialize the snapshot tree in recovery mode to handle this special case:
 	// - Users run the `prune-state` command multiple times
@@ -362,7 +363,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
 	// - The state HEAD is rewound already because of multiple incomplete `prune-state`
 	// In this case, even the state HEAD is not exactly matched with snapshot, it
 	// still feasible to recover the pruning correctly.
-	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headHeader.Root, false, false, true)
+	snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, true)
 	if err != nil {
 		return err // The relevant snapshot(s) might not exist
 	}
@@ -382,7 +383,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
 	// otherwise the dangling state will be left.
 	var (
 		found       bool
-		layers      = snaptree.Snapshots(headHeader.Root, 128, true)
+		layers      = snaptree.Snapshots(headBlock.Root(), 128, true)
 		middleRoots = make(map[common.Hash]struct{})
 	)
 	for _, layer := range layers {
@@ -396,28 +397,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err
 		log.Error("Pruning target state is not existent")
 		return errors.New("non-existent target state")
 	}
-	if err := prune(db, stateBloom, middleRoots, time.Now()); err != nil {
-		return err
-	}
-	// Pruning is done, now drop the "useless" layers from the snapshot.
-	// Firstly, flushing the target layer into the disk. After that all
-	// diff layers below the target will all be merged into the disk.
-	if err := snaptree.Cap(stateBloomRoot, 0); err != nil {
-		return err
-	}
-	// Secondly, flushing the snapshot journal into the disk. All diff
-	// layers upon are dropped silently. Eventually the entire snapshot
-	// tree is converted into a single disk layer with the pruning target
-	// as the root.
-	if _, err := snaptree.Journal(stateBloomRoot); err != nil {
-		return err
-	}
-	// Delete the state bloom, it marks the entire pruning procedure is
-	// finished. If any crashes or manual exit happens before this,
-	// `RecoverPruning` will pick it up in the next restarts to redo all
-	// the things.
-	os.RemoveAll(stateBloomPath)
-	return nil
+	return prune(snaptree, stateBloomRoot, db, stateBloom, stateBloomPath, middleRoots, time.Now())
 }
 
 // extractGenesis loads the genesis state and commits all the state entries
@@ -506,22 +486,6 @@ func findBloomFilter(datadir string) (string, common.Hash, error) {
 	return stateBloomPath, stateBloomRoot, nil
 }
 
-func getHeadHeader(db ethdb.Database) (*types.Header, error) {
-	headHeaderHash := rawdb.ReadHeadBlockHash(db)
-	if headHeaderHash == (common.Hash{}) {
-		return nil, errors.New("empty head block hash")
-	}
-	headHeaderNumber := rawdb.ReadHeaderNumber(db, headHeaderHash)
-	if headHeaderNumber == nil {
-		return nil, errors.New("empty head block number")
-	}
-	headHeader := rawdb.ReadHeader(db, headHeaderHash, *headHeaderNumber)
-	if headHeader == nil {
-		return nil, errors.New("empty head header")
-	}
-	return headHeader, nil
-}
-
 const warningLog = `
 
 WARNING!

core/state/snapshot/conversion.go

@@ -322,7 +322,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash,
 						return
 					}
 					if !bytes.Equal(account.Root, subroot.Bytes()) {
-						results <- fmt.Errorf("invalid subroot(%x), want %x, got %x", it.Hash(), account.Root, subroot)
+						results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot)
 						return
 					}
 					results <- nil

core/state/snapshot/difflayer.go

@@ -296,13 +296,17 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
 	if !hit {
 		hit = dl.diffed.Contains(destructBloomHasher(hash))
 	}
+	var origin *diskLayer
+	if !hit {
+		origin = dl.origin // extract origin while holding the lock
+	}
 	dl.lock.RUnlock()
 
 	// If the bloom filter misses, don't even bother with traversing the memory
 	// diff layers, reach straight into the bottom persistent disk layer
-	if !hit {
+	if origin != nil {
 		snapshotBloomAccountMissMeter.Mark(1)
-		return dl.origin.AccountRLP(hash)
+		return origin.AccountRLP(hash)
 	}
 	// The bloom filter hit, start poking in the internal maps
 	return dl.accountRLP(hash, 0)
@@ -358,13 +362,17 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro
 	if !hit {
 		hit = dl.diffed.Contains(destructBloomHasher(accountHash))
 	}
+	var origin *diskLayer
+	if !hit {
+		origin = dl.origin // extract origin while holding the lock
+	}
 	dl.lock.RUnlock()
 
 	// If the bloom filter misses, don't even bother with traversing the memory
 	// diff layers, reach straight into the bottom persistent disk layer
-	if !hit {
+	if origin != nil {
 		snapshotBloomStorageMissMeter.Mark(1)
-		return dl.origin.Storage(accountHash, storageHash)
+		return origin.Storage(accountHash, storageHash)
 	}
 	// The bloom filter hit, start poking in the internal maps
 	return dl.storage(accountHash, storageHash, 0)

core/state/snapshot/disklayer_test.go

@@ -524,7 +524,7 @@ func TestDiskSeek(t *testing.T) {
 		t.Fatal(err)
 	} else {
 		defer os.RemoveAll(dir)
-		diskdb, err := leveldb.New(dir, 256, 0, "")
+		diskdb, err := leveldb.New(dir, 256, 0, "", false)
 		if err != nil {
 			t.Fatal(err)
 		}

core/state/snapshot/generate.go

@@ -19,17 +19,21 @@ package snapshot
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"math/big"
 	"time"
 
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 )
@@ -40,17 +44,63 @@ var (
 
 	// emptyCode is the known hash of the empty EVM bytecode.
 	emptyCode = crypto.Keccak256Hash(nil)
+
+	// accountCheckRange is the upper limit of the number of accounts involved in
+	// each range check. This is a value estimated based on experience. If this
+	// value is too large, the failure rate of range prove will increase. Otherwise
+	// the the value is too small, the efficiency of the state recovery will decrease.
+	accountCheckRange = 128
+
+	// storageCheckRange is the upper limit of the number of storage slots involved
+	// in each range check. This is a value estimated based on experience. If this
+	// value is too large, the failure rate of range prove will increase. Otherwise
+	// the the value is too small, the efficiency of the state recovery will decrease.
+	storageCheckRange = 1024
+
+	// errMissingTrie is returned if the target trie is missing while the generation
+	// is running. In this case the generation is aborted and wait the new signal.
+	errMissingTrie = errors.New("missing trie")
+)
+
+// Metrics in generation
+var (
+	snapGeneratedAccountMeter     = metrics.NewRegisteredMeter("state/snapshot/generation/account/generated", nil)
+	snapRecoveredAccountMeter     = metrics.NewRegisteredMeter("state/snapshot/generation/account/recovered", nil)
+	snapWipedAccountMeter         = metrics.NewRegisteredMeter("state/snapshot/generation/account/wiped", nil)
+	snapMissallAccountMeter       = metrics.NewRegisteredMeter("state/snapshot/generation/account/missall", nil)
+	snapGeneratedStorageMeter     = metrics.NewRegisteredMeter("state/snapshot/generation/storage/generated", nil)
+	snapRecoveredStorageMeter     = metrics.NewRegisteredMeter("state/snapshot/generation/storage/recovered", nil)
+	snapWipedStorageMeter         = metrics.NewRegisteredMeter("state/snapshot/generation/storage/wiped", nil)
+	snapMissallStorageMeter       = metrics.NewRegisteredMeter("state/snapshot/generation/storage/missall", nil)
+	snapSuccessfulRangeProofMeter = metrics.NewRegisteredMeter("state/snapshot/generation/proof/success", nil)
+	snapFailedRangeProofMeter     = metrics.NewRegisteredMeter("state/snapshot/generation/proof/failure", nil)
+
+	// snapAccountProveCounter measures time spent on the account proving
+	snapAccountProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/prove", nil)
+	// snapAccountTrieReadCounter measures time spent on the account trie iteration
+	snapAccountTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/trieread", nil)
+	// snapAccountSnapReadCounter measues time spent on the snapshot account iteration
+	snapAccountSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/snapread", nil)
+	// snapAccountWriteCounter measures time spent on writing/updating/deleting accounts
+	snapAccountWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/account/write", nil)
+	// snapStorageProveCounter measures time spent on storage proving
+	snapStorageProveCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/prove", nil)
+	// snapStorageTrieReadCounter measures time spent on the storage trie iteration
+	snapStorageTrieReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/trieread", nil)
+	// snapStorageSnapReadCounter measures time spent on the snapshot storage iteration
+	snapStorageSnapReadCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/snapread", nil)
+	// snapStorageWriteCounter measures time spent on writing/updating/deleting storages
+	snapStorageWriteCounter = metrics.NewRegisteredCounter("state/snapshot/generation/duration/storage/write", nil)
 )
 
 // generatorStats is a collection of statistics gathered by the snapshot generator
 // for logging purposes.
 type generatorStats struct {
-	wiping   chan struct{}      // Notification channel if wiping is in progress
 	origin   uint64             // Origin prefix where generation started
 	start    time.Time          // Timestamp when generation started
-	accounts uint64             // Number of accounts indexed
-	slots    uint64             // Number of storage slots indexed
-	storage  common.StorageSize // Account and storage slot size
+	accounts uint64             // Number of accounts indexed(generated or recovered)
+	slots    uint64             // Number of storage slots indexed(generated or recovered)
+	storage  common.StorageSize // Total account and storage slot size(generation or recovery)
 }
 
 // Log creates an contextual log with the given message and the context pulled
@@ -94,22 +144,17 @@ func (gs *generatorStats) Log(msg string, root common.Hash, marker []byte) {
 // generateSnapshot regenerates a brand new snapshot based on an existing state
 // database and head block asynchronously. The snapshot is returned immediately
 // and generation is continued in the background until done.
-func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, wiper chan struct{}) *diskLayer {
-	// Wipe any previously existing snapshot from the database if no wiper is
-	// currently in progress.
-	if wiper == nil {
-		wiper = wipeSnapshot(diskdb, true)
-	}
+func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *diskLayer {
 	// Create a new disk layer with an initialized state marker at zero
 	var (
-		stats     = &generatorStats{wiping: wiper, start: time.Now()}
+		stats     = &generatorStats{start: time.Now()}
 		batch     = diskdb.NewBatch()
 		genMarker = []byte{} // Initialized but empty!
 	)
 	rawdb.WriteSnapshotRoot(batch, root)
 	journalProgress(batch, genMarker, stats)
 	if err := batch.Write(); err != nil {
-		log.Crit("Failed to write initialized state marker", "error", err)
+		log.Crit("Failed to write initialized state marker", "err", err)
 	}
 	base := &diskLayer{
 		diskdb:     diskdb,
@@ -135,7 +180,6 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta
 		Marker: marker,
 	}
 	if stats != nil {
-		entry.Wiping = (stats.wiping != nil)
 		entry.Accounts = stats.accounts
 		entry.Slots = stats.slots
 		entry.Storage = uint64(stats.storage)
@@ -159,169 +203,538 @@ func journalProgress(db ethdb.KeyValueWriter, marker []byte, stats *generatorSta
 	rawdb.WriteSnapshotGenerator(db, blob)
 }
 
+// proofResult contains the output of range proving which can be used
+// for further processing regardless if it is successful or not.
+type proofResult struct {
+	keys     [][]byte   // The key set of all elements being iterated, even proving is failed
+	vals     [][]byte   // The val set of all elements being iterated, even proving is failed
+	diskMore bool       // Set when the database has extra snapshot states since last iteration
+	trieMore bool       // Set when the trie has extra snapshot states(only meaningful for successful proving)
+	proofErr error      // Indicator whether the given state range is valid or not
+	tr       *trie.Trie // The trie, in case the trie was resolved by the prover (may be nil)
+}
+
+// valid returns the indicator that range proof is successful or not.
+func (result *proofResult) valid() bool {
+	return result.proofErr == nil
+}
+
+// last returns the last verified element key regardless of whether the range proof is
+// successful or not. Nil is returned if nothing involved in the proving.
+func (result *proofResult) last() []byte {
+	var last []byte
+	if len(result.keys) > 0 {
+		last = result.keys[len(result.keys)-1]
+	}
+	return last
+}
+
+// forEach iterates all the visited elements and applies the given callback on them.
+// The iteration is aborted if the callback returns non-nil error.
+func (result *proofResult) forEach(callback func(key []byte, val []byte) error) error {
+	for i := 0; i < len(result.keys); i++ {
+		key, val := result.keys[i], result.vals[i]
+		if err := callback(key, val); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// proveRange proves the snapshot segment with particular prefix is "valid".
+// The iteration start point will be assigned if the iterator is restored from
+// the last interruption. Max will be assigned in order to limit the maximum
+// amount of data involved in each iteration.
+//
+// The proof result will be returned if the range proving is finished, otherwise
+// the error will be returned to abort the entire procedure.
+func (dl *diskLayer) proveRange(stats *generatorStats, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
+	var (
+		keys     [][]byte
+		vals     [][]byte
+		proof    = rawdb.NewMemoryDatabase()
+		diskMore = false
+	)
+	iter := dl.diskdb.NewIterator(prefix, origin)
+	defer iter.Release()
+
+	var start = time.Now()
+	for iter.Next() {
+		key := iter.Key()
+		if len(key) != len(prefix)+common.HashLength {
+			continue
+		}
+		if len(keys) == max {
+			// Break if we've reached the max size, and signal that we're not
+			// done yet.
+			diskMore = true
+			break
+		}
+		keys = append(keys, common.CopyBytes(key[len(prefix):]))
+
+		if valueConvertFn == nil {
+			vals = append(vals, common.CopyBytes(iter.Value()))
+		} else {
+			val, err := valueConvertFn(iter.Value())
+			if err != nil {
+				// Special case, the state data is corrupted (invalid slim-format account),
+				// don't abort the entire procedure directly. Instead, let the fallback
+				// generation to heal the invalid data.
+				//
+				// Here append the original value to ensure that the number of key and
+				// value are the same.
+				vals = append(vals, common.CopyBytes(iter.Value()))
+				log.Error("Failed to convert account state data", "err", err)
+			} else {
+				vals = append(vals, val)
+			}
+		}
+	}
+	// Update metrics for database iteration and merkle proving
+	if kind == "storage" {
+		snapStorageSnapReadCounter.Inc(time.Since(start).Nanoseconds())
+	} else {
+		snapAccountSnapReadCounter.Inc(time.Since(start).Nanoseconds())
+	}
+	defer func(start time.Time) {
+		if kind == "storage" {
+			snapStorageProveCounter.Inc(time.Since(start).Nanoseconds())
+		} else {
+			snapAccountProveCounter.Inc(time.Since(start).Nanoseconds())
+		}
+	}(time.Now())
+
+	// The snap state is exhausted, pass the entire key/val set for verification
+	if origin == nil && !diskMore {
+		stackTr := trie.NewStackTrie(nil)
+		for i, key := range keys {
+			stackTr.TryUpdate(key, vals[i])
+		}
+		if gotRoot := stackTr.Hash(); gotRoot != root {
+			return &proofResult{
+				keys:     keys,
+				vals:     vals,
+				proofErr: fmt.Errorf("wrong root: have %#x want %#x", gotRoot, root),
+			}, nil
+		}
+		return &proofResult{keys: keys, vals: vals}, nil
+	}
+	// Snap state is chunked, generate edge proofs for verification.
+	tr, err := trie.New(root, dl.triedb)
+	if err != nil {
+		stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
+		return nil, errMissingTrie
+	}
+	// Firstly find out the key of last iterated element.
+	var last []byte
+	if len(keys) > 0 {
+		last = keys[len(keys)-1]
+	}
+	// Generate the Merkle proofs for the first and last element
+	if origin == nil {
+		origin = common.Hash{}.Bytes()
+	}
+	if err := tr.Prove(origin, 0, proof); err != nil {
+		log.Debug("Failed to prove range", "kind", kind, "origin", origin, "err", err)
+		return &proofResult{
+			keys:     keys,
+			vals:     vals,
+			diskMore: diskMore,
+			proofErr: err,
+			tr:       tr,
+		}, nil
+	}
+	if last != nil {
+		if err := tr.Prove(last, 0, proof); err != nil {
+			log.Debug("Failed to prove range", "kind", kind, "last", last, "err", err)
+			return &proofResult{
+				keys:     keys,
+				vals:     vals,
+				diskMore: diskMore,
+				proofErr: err,
+				tr:       tr,
+			}, nil
+		}
+	}
+	// Verify the snapshot segment with range prover, ensure that all flat states
+	// in this range correspond to merkle trie.
+	cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof)
+	return &proofResult{
+			keys:     keys,
+			vals:     vals,
+			diskMore: diskMore,
+			trieMore: cont,
+			proofErr: err,
+			tr:       tr},
+		nil
+}
+
+// onStateCallback is a function that is called by generateRange, when processing a range of
+// accounts or storage slots. For each element, the callback is invoked.
+// If 'delete' is true, then this element (and potential slots) needs to be deleted from the snapshot.
+// If 'write' is true, then this element needs to be updated with the 'val'.
+// If 'write' is false, then this element is already correct, and needs no update. However,
+// for accounts, the storage trie of the account needs to be checked.
+// The 'val' is the canonical encoding of the value (not the slim format for accounts)
+type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
+
+// generateRange generates the state segment with particular prefix. Generation can
+// either verify the correctness of existing state through rangeproof and skip
+// generation, or iterate trie to regenerate state on demand.
+func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, origin []byte, max int, stats *generatorStats, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
+	// Use range prover to check the validity of the flat state in the range
+	result, err := dl.proveRange(stats, root, prefix, kind, origin, max, valueConvertFn)
+	if err != nil {
+		return false, nil, err
+	}
+	last := result.last()
+
+	// Construct contextual logger
+	logCtx := []interface{}{"kind", kind, "prefix", hexutil.Encode(prefix)}
+	if len(origin) > 0 {
+		logCtx = append(logCtx, "origin", hexutil.Encode(origin))
+	}
+	logger := log.New(logCtx...)
+
+	// The range prover says the range is correct, skip trie iteration
+	if result.valid() {
+		snapSuccessfulRangeProofMeter.Mark(1)
+		logger.Trace("Proved state range", "last", hexutil.Encode(last))
+
+		// The verification is passed, process each state with the given
+		// callback function. If this state represents a contract, the
+		// corresponding storage check will be performed in the callback
+		if err := result.forEach(func(key []byte, val []byte) error { return onState(key, val, false, false) }); err != nil {
+			return false, nil, err
+		}
+		// Only abort the iteration when both database and trie are exhausted
+		return !result.diskMore && !result.trieMore, last, nil
+	}
+	logger.Trace("Detected outdated state range", "last", hexutil.Encode(last), "err", result.proofErr)
+	snapFailedRangeProofMeter.Mark(1)
+
+	// Special case, the entire trie is missing. In the original trie scheme,
+	// all the duplicated subtries will be filter out(only one copy of data
+	// will be stored). While in the snapshot model, all the storage tries
+	// belong to different contracts will be kept even they are duplicated.
+	// Track it to a certain extent remove the noise data used for statistics.
+	if origin == nil && last == nil {
+		meter := snapMissallAccountMeter
+		if kind == "storage" {
+			meter = snapMissallStorageMeter
+		}
+		meter.Mark(1)
+	}
+
+	// We use the snap data to build up a cache which can be used by the
+	// main account trie as a primary lookup when resolving hashes
+	var snapNodeCache ethdb.KeyValueStore
+	if len(result.keys) > 0 {
+		snapNodeCache = memorydb.New()
+		snapTrieDb := trie.NewDatabase(snapNodeCache)
+		snapTrie, _ := trie.New(common.Hash{}, snapTrieDb)
+		for i, key := range result.keys {
+			snapTrie.Update(key, result.vals[i])
+		}
+		root, _ := snapTrie.Commit(nil)
+		snapTrieDb.Commit(root, false, nil)
+	}
+	tr := result.tr
+	if tr == nil {
+		tr, err = trie.New(root, dl.triedb)
+		if err != nil {
+			stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
+			return false, nil, errMissingTrie
+		}
+	}
+
+	var (
+		trieMore       bool
+		nodeIt         = tr.NodeIterator(origin)
+		iter           = trie.NewIterator(nodeIt)
+		kvkeys, kvvals = result.keys, result.vals
+
+		// counters
+		count     = 0 // number of states delivered by iterator
+		created   = 0 // states created from the trie
+		updated   = 0 // states updated from the trie
+		deleted   = 0 // states not in trie, but were in snapshot
+		untouched = 0 // states already correct
+
+		// timers
+		start    = time.Now()
+		internal time.Duration
+	)
+	nodeIt.AddResolver(snapNodeCache)
+	for iter.Next() {
+		if last != nil && bytes.Compare(iter.Key, last) > 0 {
+			trieMore = true
+			break
+		}
+		count++
+		write := true
+		created++
+		for len(kvkeys) > 0 {
+			if cmp := bytes.Compare(kvkeys[0], iter.Key); cmp < 0 {
+				// delete the key
+				istart := time.Now()
+				if err := onState(kvkeys[0], nil, false, true); err != nil {
+					return false, nil, err
+				}
+				kvkeys = kvkeys[1:]
+				kvvals = kvvals[1:]
+				deleted++
+				internal += time.Since(istart)
+				continue
+			} else if cmp == 0 {
+				// the snapshot key can be overwritten
+				created--
+				if write = !bytes.Equal(kvvals[0], iter.Value); write {
+					updated++
+				} else {
+					untouched++
+				}
+				kvkeys = kvkeys[1:]
+				kvvals = kvvals[1:]
+			}
+			break
+		}
+		istart := time.Now()
+		if err := onState(iter.Key, iter.Value, write, false); err != nil {
+			return false, nil, err
+		}
+		internal += time.Since(istart)
+	}
+	if iter.Err != nil {
+		return false, nil, iter.Err
+	}
+	// Delete all stale snapshot states remaining
+	istart := time.Now()
+	for _, key := range kvkeys {
+		if err := onState(key, nil, false, true); err != nil {
+			return false, nil, err
+		}
+		deleted += 1
+	}
+	internal += time.Since(istart)
+
+	// Update metrics for counting trie iteration
+	if kind == "storage" {
+		snapStorageTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
+	} else {
+		snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
+	}
+	logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last),
+		"count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted)
+
+	// If there are either more trie items, or there are more snap items
+	// (in the next segment), then we need to keep working
+	return !trieMore && !result.diskMore, last, nil
+}
+
 // generate is a background thread that iterates over the state and storage tries,
 // constructing the state snapshot. All the arguments are purely for statistics
 // gathering and logging, since the method surfs the blocks as they arrive, often
 // being restarted.
 func (dl *diskLayer) generate(stats *generatorStats) {
-	// If a database wipe is in operation, wait until it's done
-	if stats.wiping != nil {
-		stats.Log("Wiper running, state snapshotting paused", common.Hash{}, dl.genMarker)
-		select {
-		// If wiper is done, resume normal mode of operation
-		case <-stats.wiping:
-			stats.wiping = nil
-			stats.start = time.Now()
-
-		// If generator was aborted during wipe, return
-		case abort := <-dl.genAbort:
-			abort <- stats
-			return
-		}
-	}
-	// Create an account and state iterator pointing to the current generator marker
-	accTrie, err := trie.NewSecure(dl.root, dl.triedb)
-	if err != nil {
-		// The account trie is missing (GC), surf the chain until one becomes available
-		stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
-
-		abort := <-dl.genAbort
-		abort <- stats
-		return
+	var (
+		accMarker    []byte
+		accountRange = accountCheckRange
+	)
+	if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
+		// Always reset the initial account range as 1
+		// whenever recover from the interruption.
+		accMarker, accountRange = dl.genMarker[:common.HashLength], 1
 	}
+	var (
+		batch     = dl.diskdb.NewBatch()
+		logged    = time.Now()
+		accOrigin = common.CopyBytes(accMarker)
+		abort     chan *generatorStats
+	)
 	stats.Log("Resuming state snapshot generation", dl.root, dl.genMarker)
 
-	var accMarker []byte
-	if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
-		accMarker = dl.genMarker[:common.HashLength]
+	checkAndFlush := func(currentLocation []byte) error {
+		select {
+		case abort = <-dl.genAbort:
+		default:
 		}
-	accIt := trie.NewIterator(accTrie.NodeIterator(accMarker))
-	batch := dl.diskdb.NewBatch()
+		if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
+			// Flush out the batch anyway no matter it's empty or not.
+			// It's possible that all the states are recovered and the
+			// generation indeed makes progress.
+			journalProgress(batch, currentLocation, stats)
 
-	// Iterate from the previous marker and continue generating the state snapshot
-	logged := time.Now()
-	for accIt.Next() {
+			if err := batch.Write(); err != nil {
+				return err
+			}
+			batch.Reset()
+
+			dl.lock.Lock()
+			dl.genMarker = currentLocation
+			dl.lock.Unlock()
+
+			if abort != nil {
+				stats.Log("Aborting state snapshot generation", dl.root, currentLocation)
+				return errors.New("aborted")
+			}
+		}
+		if time.Since(logged) > 8*time.Second {
+			stats.Log("Generating state snapshot", dl.root, currentLocation)
+			logged = time.Now()
+		}
+		return nil
+	}
+
+	onAccount := func(key []byte, val []byte, write bool, delete bool) error {
+		var (
+			start       = time.Now()
+			accountHash = common.BytesToHash(key)
+		)
+		if delete {
+			rawdb.DeleteAccountSnapshot(batch, accountHash)
+			snapWipedAccountMeter.Mark(1)
+
+			// Ensure that any previous snapshot storage values are cleared
+			prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...)
+			keyLen := len(rawdb.SnapshotStoragePrefix) + 2*common.HashLength
+			if err := wipeKeyRange(dl.diskdb, "storage", prefix, nil, nil, keyLen, snapWipedStorageMeter, false); err != nil {
+				return err
+			}
+			snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds())
+			return nil
+		}
 		// Retrieve the current account and flatten it into the internal format
-		accountHash := common.BytesToHash(accIt.Key)
-
 		var acc struct {
 			Nonce    uint64
 			Balance  *big.Int
 			Root     common.Hash
 			CodeHash []byte
 		}
-		if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil {
+		if err := rlp.DecodeBytes(val, &acc); err != nil {
 			log.Crit("Invalid account encountered during snapshot creation", "err", err)
 		}
-		data := SlimAccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash)
-
 		// If the account is not yet in-progress, write it out
 		if accMarker == nil || !bytes.Equal(accountHash[:], accMarker) {
+			dataLen := len(val) // Approximate size, saves us a round of RLP-encoding
+			if !write {
+				if bytes.Equal(acc.CodeHash, emptyCode[:]) {
+					dataLen -= 32
+				}
+				if acc.Root == emptyRoot {
+					dataLen -= 32
+				}
+				snapRecoveredAccountMeter.Mark(1)
+			} else {
+				data := SlimAccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash)
+				dataLen = len(data)
 				rawdb.WriteAccountSnapshot(batch, accountHash, data)
-			stats.storage += common.StorageSize(1 + common.HashLength + len(data))
+				snapGeneratedAccountMeter.Mark(1)
+			}
+			stats.storage += common.StorageSize(1 + common.HashLength + dataLen)
 			stats.accounts++
 		}
 		// If we've exceeded our batch allowance or termination was requested, flush to disk
-		var abort chan *generatorStats
-		select {
-		case abort = <-dl.genAbort:
-		default:
+		if err := checkAndFlush(accountHash[:]); err != nil {
+			return err
 		}
-		if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
-			// Only write and set the marker if we actually did something useful
-			if batch.ValueSize() > 0 {
-				// Ensure the generator entry is in sync with the data
-				marker := accountHash[:]
-				journalProgress(batch, marker, stats)
+		// If the iterated account is the contract, create a further loop to
+		// verify or regenerate the contract storage.
+		if acc.Root == emptyRoot {
+			// If the root is empty, we still need to ensure that any previous snapshot
+			// storage values are cleared
+			// TODO: investigate if this can be avoided, this will be very costly since it
+			// affects every single EOA account
+			//  - Perhaps we can avoid if where codeHash is emptyCode
+			prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...)
+			keyLen := len(rawdb.SnapshotStoragePrefix) + 2*common.HashLength
+			if err := wipeKeyRange(dl.diskdb, "storage", prefix, nil, nil, keyLen, snapWipedStorageMeter, false); err != nil {
+				return err
+			}
+			snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds())
+		} else {
+			snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds())
 
-				batch.Write()
-				batch.Reset()
-
-				dl.lock.Lock()
-				dl.genMarker = marker
-				dl.lock.Unlock()
-			}
-			if abort != nil {
-				stats.Log("Aborting state snapshot generation", dl.root, accountHash[:])
-				abort <- stats
-				return
-			}
-		}
-		// If the account is in-progress, continue where we left off (otherwise iterate all)
-		if acc.Root != emptyRoot {
-			storeTrie, err := trie.NewSecure(acc.Root, dl.triedb)
-			if err != nil {
-				log.Error("Generator failed to access storage trie", "root", dl.root, "account", accountHash, "stroot", acc.Root, "err", err)
-				abort := <-dl.genAbort
-				abort <- stats
-				return
-			}
 			var storeMarker []byte
 			if accMarker != nil && bytes.Equal(accountHash[:], accMarker) && len(dl.genMarker) > common.HashLength {
 				storeMarker = dl.genMarker[common.HashLength:]
 			}
-			storeIt := trie.NewIterator(storeTrie.NodeIterator(storeMarker))
-			for storeIt.Next() {
-				rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(storeIt.Key), storeIt.Value)
-				stats.storage += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value))
+			onStorage := func(key []byte, val []byte, write bool, delete bool) error {
+				defer func(start time.Time) {
+					snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds())
+				}(time.Now())
+
+				if delete {
+					rawdb.DeleteStorageSnapshot(batch, accountHash, common.BytesToHash(key))
+					snapWipedStorageMeter.Mark(1)
+					return nil
+				}
+				if write {
+					rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(key), val)
+					snapGeneratedStorageMeter.Mark(1)
+				} else {
+					snapRecoveredStorageMeter.Mark(1)
+				}
+				stats.storage += common.StorageSize(1 + 2*common.HashLength + len(val))
 				stats.slots++
 
 				// If we've exceeded our batch allowance or termination was requested, flush to disk
-				var abort chan *generatorStats
-				select {
-				case abort = <-dl.genAbort:
-				default:
+				if err := checkAndFlush(append(accountHash[:], key...)); err != nil {
+					return err
 				}
-				if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
-					// Only write and set the marker if we actually did something useful
-					if batch.ValueSize() > 0 {
-						// Ensure the generator entry is in sync with the data
-						marker := append(accountHash[:], storeIt.Key...)
-						journalProgress(batch, marker, stats)
-
-						batch.Write()
-						batch.Reset()
-
-						dl.lock.Lock()
-						dl.genMarker = marker
-						dl.lock.Unlock()
+				return nil
 			}
-					if abort != nil {
-						stats.Log("Aborting state snapshot generation", dl.root, append(accountHash[:], storeIt.Key...))
-						abort <- stats
-						return
+			var storeOrigin = common.CopyBytes(storeMarker)
+			for {
+				exhausted, last, err := dl.generateRange(acc.Root, append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...), "storage", storeOrigin, storageCheckRange, stats, onStorage, nil)
+				if err != nil {
+					return err
 				}
-					if time.Since(logged) > 8*time.Second {
-						stats.Log("Generating state snapshot", dl.root, append(accountHash[:], storeIt.Key...))
-						logged = time.Now()
+				if exhausted {
+					break
+				}
+				if storeOrigin = increaseKey(last); storeOrigin == nil {
+					break // special case, the last is 0xffffffff...fff
 				}
 			}
 		}
-			if err := storeIt.Err; err != nil {
-				log.Error("Generator failed to iterate storage trie", "accroot", dl.root, "acchash", common.BytesToHash(accIt.Key), "stroot", acc.Root, "err", err)
-				abort := <-dl.genAbort
-				abort <- stats
-				return
-			}
-		}
-		if time.Since(logged) > 8*time.Second {
-			stats.Log("Generating state snapshot", dl.root, accIt.Key)
-			logged = time.Now()
-		}
 		// Some account processed, unmark the marker
 		accMarker = nil
+		return nil
+	}
+
+	// Global loop for regerating the entire state trie + all layered storage tries.
+	for {
+		exhausted, last, err := dl.generateRange(dl.root, rawdb.SnapshotAccountPrefix, "account", accOrigin, accountRange, stats, onAccount, FullAccountRLP)
+		// The procedure it aborted, either by external signal or internal error
+		if err != nil {
+			if abort == nil { // aborted by internal error, wait the signal
+				abort = <-dl.genAbort
 			}
-	if err := accIt.Err; err != nil {
-		log.Error("Generator failed to iterate account trie", "root", dl.root, "err", err)
-		abort := <-dl.genAbort
 			abort <- stats
 			return
 		}
+		// Abort the procedure if the entire snapshot is generated
+		if exhausted {
+			break
+		}
+		if accOrigin = increaseKey(last); accOrigin == nil {
+			break // special case, the last is 0xffffffff...fff
+		}
+		accountRange = accountCheckRange
+	}
 	// Snapshot fully generated, set the marker to nil.
 	// Note even there is nothing to commit, persist the
 	// generator anyway to mark the snapshot is complete.
 	journalProgress(batch, nil, stats)
-	batch.Write()
+	if err := batch.Write(); err != nil {
+		log.Error("Failed to flush batch", "err", err)
+
+		abort = <-dl.genAbort
+		abort <- stats
+		return
+	}
+	batch.Reset()
 
 	log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots,
 		"storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start)))
@@ -332,6 +745,18 @@ func (dl *diskLayer) generate(stats *generatorStats) {
 	dl.lock.Unlock()
 
 	// Someone will be looking for us, wait it out
-	abort := <-dl.genAbort
+	abort = <-dl.genAbort
 	abort <- nil
 }
+
+// increaseKey increase the input key by one bit. Return nil if the entire
+// addition operation overflows,
+func increaseKey(key []byte) []byte {
+	for i := len(key) - 1; i >= 0; i-- {
+		key[i]++
+		if key[i] != 0x0 {
+			return key
+		}
+	}
+	return nil
+}

core/state/snapshot/generate_test.go

@@ -17,16 +17,361 @@
 package snapshot
 
 import (
+	"fmt"
 	"math/big"
+	"os"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	"golang.org/x/crypto/sha3"
 )
 
+// Tests that snapshot generation from an empty database.
+func TestGeneration(t *testing.T) {
+	// We can't use statedb to make a test trie (circular dependency), so make
+	// a fake one manually. We're going with a small account trie of 3 accounts,
+	// two of which also has the same 3-slot storage trie attached.
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+	)
+	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
+	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
+	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
+	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+
+	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ := rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+
+	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ = rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+
+	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ = rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
+	root, _ := accTrie.Commit(nil)       // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
+	triedb.Commit(root, false, nil)
+
+	if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want {
+		t.Fatalf("have %#x want %#x", have, want)
+	}
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+func hashData(input []byte) common.Hash {
+	var hasher = sha3.NewLegacyKeccak256()
+	var hash common.Hash
+	hasher.Reset()
+	hasher.Write(input)
+	hasher.Sum(hash[:0])
+	return hash
+}
+
+// Tests that snapshot generation with existent flat state.
+func TestGenerateExistentState(t *testing.T) {
+	// We can't use statedb to make a test trie (circular dependency), so make
+	// a fake one manually. We're going with a small account trie of 3 accounts,
+	// two of which also has the same 3-slot storage trie attached.
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+	)
+	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
+	stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
+	stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
+	stTrie.Commit(nil)                              // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+
+	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ := rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val)
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1"))
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2"))
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3"))
+
+	acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ = rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+	diskdb.Put(hashData([]byte("acc-2")).Bytes(), val)
+	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val)
+
+	acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+	val, _ = rlp.EncodeToBytes(acc)
+	accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
+	rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val)
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1"))
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2"))
+	rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3"))
+
+	root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd
+	triedb.Commit(root, false, nil)
+
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
+	t.Helper()
+	accIt := snap.AccountIterator(common.Hash{})
+	defer accIt.Release()
+	snapRoot, err := generateTrieRoot(nil, accIt, common.Hash{}, stackTrieGenerate,
+		func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) {
+			storageIt, _ := snap.StorageIterator(accountHash, common.Hash{})
+			defer storageIt.Release()
+
+			hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false)
+			if err != nil {
+				return common.Hash{}, err
+			}
+			return hash, nil
+		}, newGenerateStats(), true)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+	if snapRoot != trieRoot {
+		t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot)
+	}
+}
+
+type testHelper struct {
+	diskdb  *memorydb.Database
+	triedb  *trie.Database
+	accTrie *trie.SecureTrie
+}
+
+func newHelper() *testHelper {
+	diskdb := memorydb.New()
+	triedb := trie.NewDatabase(diskdb)
+	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	return &testHelper{
+		diskdb:  diskdb,
+		triedb:  triedb,
+		accTrie: accTrie,
+	}
+}
+
+func (t *testHelper) addTrieAccount(acckey string, acc *Account) {
+	val, _ := rlp.EncodeToBytes(acc)
+	t.accTrie.Update([]byte(acckey), val)
+}
+
+func (t *testHelper) addSnapAccount(acckey string, acc *Account) {
+	val, _ := rlp.EncodeToBytes(acc)
+	key := hashData([]byte(acckey))
+	rawdb.WriteAccountSnapshot(t.diskdb, key, val)
+}
+
+func (t *testHelper) addAccount(acckey string, acc *Account) {
+	t.addTrieAccount(acckey, acc)
+	t.addSnapAccount(acckey, acc)
+}
+
+func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) {
+	accHash := hashData([]byte(accKey))
+	for i, key := range keys {
+		rawdb.WriteStorageSnapshot(t.diskdb, accHash, hashData([]byte(key)), []byte(vals[i]))
+	}
+}
+
+func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte {
+	stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb)
+	for i, k := range keys {
+		stTrie.Update([]byte(k), []byte(vals[i]))
+	}
+	root, _ := stTrie.Commit(nil)
+	return root.Bytes()
+}
+
+func (t *testHelper) Generate() (common.Hash, *diskLayer) {
+	root, _ := t.accTrie.Commit(nil)
+	t.triedb.Commit(root, false, nil)
+	snap := generateSnapshot(t.diskdb, t.triedb, 16, root)
+	return root, snap
+}
+
+// Tests that snapshot generation with existent flat state, where the flat state
+// contains some errors:
+// - the contract with empty storage root but has storage entries in the disk
+// - the contract with non empty storage root but empty storage slots
+// - the contract(non-empty storage) misses some storage slots
+//   - miss in the beginning
+//   - miss in the middle
+//   - miss in the end
+// - the contract(non-empty storage) has wrong storage slots
+//   - wrong slots in the beginning
+//   - wrong slots in the middle
+//   - wrong slots in the end
+// - the contract(non-empty storage) has extra storage slots
+//   - extra slots in the beginning
+//   - extra slots in the middle
+//   - extra slots in the end
+func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
+	helper := newHelper()
+	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	// Account one, empty root but non-empty database
+	helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+	helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	// Account two, non empty root but empty database
+	helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+
+	// Miss slots
+	{
+		// Account three, non empty root but misses slots in the beginning
+		helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"})
+
+		// Account four, non empty root but misses slots in the middle
+		helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"})
+
+		// Account five, non empty root but misses slots in the end
+		helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"})
+	}
+
+	// Wrong storage slots
+	{
+		// Account six, non empty root but wrong slots in the beginning
+		helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"})
+
+		// Account seven, non empty root but wrong slots in the middle
+		helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"})
+
+		// Account eight, non empty root but wrong slots in the end
+		helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"})
+
+		// Account 9, non empty root but rotated slots
+		helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"})
+	}
+
+	// Extra storage slots
+	{
+		// Account 10, non empty root but extra slots in the beginning
+		helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"})
+
+		// Account 11, non empty root but extra slots in the middle
+		helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"})
+
+		// Account 12, non empty root but extra slots in the end
+		helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"})
+	}
+
+	root, snap := helper.Generate()
+	t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0
+
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+// Tests that snapshot generation with existent flat state, where the flat state
+// contains some errors:
+// - miss accounts
+// - wrong accounts
+// - extra accounts
+func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
+	helper := newHelper()
+	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+
+	// Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6]
+	// Extra accounts [acc-0, acc-5, acc-7]
+
+	// Missing accounts, only in the trie
+	{
+		helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Beginning
+		helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Middle
+		helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // End
+	}
+
+	// Wrong accounts
+	{
+		helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")})
+
+		helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+	}
+
+	// Extra accounts, only in the snap
+	{
+		helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyRoot.Bytes()})                     // before the beginning
+		helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle
+		helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()})          // after the end
+	}
+
+	root, snap := helper.Generate()
+	t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8
+
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
 // Tests that snapshot generation errors out correctly in case of a missing trie
 // node in the account trie.
 func TestGenerateCorruptAccountTrie(t *testing.T) {
@@ -55,7 +400,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 	triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil)
 	diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), nil)
+	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"))
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -115,7 +460,7 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 	// Delete a storage trie root and ensure the generator chokes
 	diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes())
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), nil)
+	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -174,7 +519,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 	// Delete a storage trie leaf and ensure the generator chokes
 	diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
 
-	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), nil)
+	snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"))
 	select {
 	case <-snap.genPending:
 		// Snapshot generation succeeded
@@ -188,3 +533,301 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 	snap.genAbort <- stop
 	<-stop
 }
+
+func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie {
+	stTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	for i := 0; i < n; i++ {
+		k := fmt.Sprintf("key-%d", i)
+		v := fmt.Sprintf("val-%d", i)
+		stTrie.Update([]byte(k), []byte(v))
+	}
+	stTrie.Commit(nil)
+	return stTrie
+}
+
+// Tests that snapshot generation when an extra account with storage exists in the snap state.
+func TestGenerateWithExtraAccounts(t *testing.T) {
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+		stTrie = getStorageTrie(5, triedb)
+	)
+	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	{ // Account one in the trie
+		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+		val, _ := rlp.EncodeToBytes(acc)
+		accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+		// Identical in the snap
+		key := hashData([]byte("acc-1"))
+		rawdb.WriteAccountSnapshot(diskdb, key, val)
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
+	}
+	{ // Account two exists only in the snapshot
+		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+		val, _ := rlp.EncodeToBytes(acc)
+		key := hashData([]byte("acc-2"))
+		rawdb.WriteAccountSnapshot(diskdb, key, val)
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
+	}
+	root, _ := accTrie.Commit(nil)
+	t.Logf("root: %x", root)
+	triedb.Commit(root, false, nil)
+	// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
+	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
+		t.Fatalf("expected snap storage to exist")
+	}
+
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+	// If we now inspect the snap db, there should exist no extraneous storage items
+	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
+		t.Fatalf("expected slot to be removed, got %v", string(data))
+	}
+}
+
+func enableLogging() {
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+}
+
+// Tests that snapshot generation when an extra account with storage exists in the snap state.
+func TestGenerateWithManyExtraAccounts(t *testing.T) {
+	if false {
+		enableLogging()
+	}
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+		stTrie = getStorageTrie(3, triedb)
+	)
+	accTrie, _ := trie.NewSecure(common.Hash{}, triedb)
+	{ // Account one in the trie
+		acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+		val, _ := rlp.EncodeToBytes(acc)
+		accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+		// Identical in the snap
+		key := hashData([]byte("acc-1"))
+		rawdb.WriteAccountSnapshot(diskdb, key, val)
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
+		rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
+	}
+	{ // 100 accounts exist only in snapshot
+		for i := 0; i < 1000; i++ {
+			//acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
+			acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
+			val, _ := rlp.EncodeToBytes(acc)
+			key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
+			rawdb.WriteAccountSnapshot(diskdb, key, val)
+		}
+	}
+	root, _ := accTrie.Commit(nil)
+	t.Logf("root: %x", root)
+	triedb.Commit(root, false, nil)
+
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+// Tests this case
+// maxAccountRange 3
+// snapshot-accounts: 01, 02, 03, 04, 05, 06, 07
+// trie-accounts:             03,             07
+//
+// We iterate three snapshot storage slots (max = 3) from the database. They are 0x01, 0x02, 0x03.
+// The trie has a lot of deletions.
+// So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
+// But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
+func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
+	accountCheckRange = 3
+	if false {
+		enableLogging()
+	}
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+	)
+	accTrie, _ := trie.New(common.Hash{}, triedb)
+	{
+		acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
+		val, _ := rlp.EncodeToBytes(acc)
+		accTrie.Update(common.HexToHash("0x03").Bytes(), val)
+		accTrie.Update(common.HexToHash("0x07").Bytes(), val)
+
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val)
+	}
+
+	root, _ := accTrie.Commit(nil)
+	t.Logf("root: %x", root)
+	triedb.Commit(root, false, nil)
+
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+// TestGenerateWithMalformedSnapdata tests what happes if we have some junk
+// in the snapshot database, which cannot be parsed back to an account
+func TestGenerateWithMalformedSnapdata(t *testing.T) {
+	accountCheckRange = 3
+	if false {
+		enableLogging()
+	}
+	var (
+		diskdb = memorydb.New()
+		triedb = trie.NewDatabase(diskdb)
+	)
+	accTrie, _ := trie.New(common.Hash{}, triedb)
+	{
+		acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
+		val, _ := rlp.EncodeToBytes(acc)
+		accTrie.Update(common.HexToHash("0x03").Bytes(), val)
+
+		junk := make([]byte, 100)
+		copy(junk, []byte{0xde, 0xad})
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk)
+		rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk)
+	}
+
+	root, _ := accTrie.Commit(nil)
+	t.Logf("root: %x", root)
+	triedb.Commit(root, false, nil)
+
+	snap := generateSnapshot(diskdb, triedb, 16, root)
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+	// If we now inspect the snap db, there should exist no extraneous storage items
+	if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
+		t.Fatalf("expected slot to be removed, got %v", string(data))
+	}
+}
+
+func TestGenerateFromEmptySnap(t *testing.T) {
+	//enableLogging()
+	accountCheckRange = 10
+	storageCheckRange = 20
+	helper := newHelper()
+	stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
+	// Add 1K accounts to the trie
+	for i := 0; i < 400; i++ {
+		helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
+			&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
+	}
+	root, snap := helper.Generate()
+	t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
+
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(1 * time.Second):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}
+
+// Tests that snapshot generation with existent flat state, where the flat state
+// storage is correct, but incomplete.
+// The incomplete part is on the second range
+// snap: [ 0x01, 0x02, 0x03, 0x04] , [ 0x05, 0x06, 0x07, {missing}] (with storageCheck = 4)
+// trie:  0x01, 0x02, 0x03, 0x04,  0x05, 0x06, 0x07, 0x08
+// This hits a case where the snap verification passes, but there are more elements in the trie
+// which we must also add.
+func TestGenerateWithIncompleteStorage(t *testing.T) {
+	storageCheckRange = 4
+	helper := newHelper()
+	stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
+	stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
+	stRoot := helper.makeStorageTrie(stKeys, stVals)
+	// We add 8 accounts, each one is missing exactly one of the storage slots. This means
+	// we don't have to order the keys and figure out exactly which hash-key winds up
+	// on the sensitive spots at the boundaries
+	for i := 0; i < 8; i++ {
+		accKey := fmt.Sprintf("acc-%d", i)
+		helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()})
+		var moddedKeys []string
+		var moddedVals []string
+		for ii := 0; ii < 8; ii++ {
+			if ii != i {
+				moddedKeys = append(moddedKeys, stKeys[ii])
+				moddedVals = append(moddedVals, stVals[ii])
+			}
+		}
+		helper.addSnapStorage(accKey, moddedKeys, moddedVals)
+	}
+
+	root, snap := helper.Generate()
+	t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
+
+	select {
+	case <-snap.genPending:
+		// Snapshot generation succeeded
+
+	case <-time.After(250 * time.Millisecond):
+		t.Errorf("Snapshot generation failed")
+	}
+	checkSnapRoot(t, snap, root)
+	// Signal abortion to the generator and wait for it to tear down
+	stop := make(chan *generatorStats)
+	snap.genAbort <- stop
+	<-stop
+}

core/state/snapshot/journal.go

@@ -37,7 +37,10 @@ const journalVersion uint64 = 0
 
 // journalGenerator is a disk layer entry containing the generator progress marker.
 type journalGenerator struct {
-	Wiping   bool // Whether the database was in progress of being wiped
+	// Indicator that whether the database was in progress of being wiped.
+	// It's deprecated but keep it here for background compatibility.
+	Wiping bool
+
 	Done     bool // Whether the generator finished creating the snapshot
 	Marker   []byte
 	Accounts uint64
@@ -63,30 +66,6 @@ type journalStorage struct {
 	Vals [][]byte
 }
 
-// loadAndParseLegacyJournal tries to parse the snapshot journal in legacy format.
-func loadAndParseLegacyJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) {
-	// Retrieve the journal, for legacy journal it must exist since even for
-	// 0 layer it stores whether we've already generated the snapshot or are
-	// in progress only.
-	journal := rawdb.ReadSnapshotJournal(db)
-	if len(journal) == 0 {
-		return nil, journalGenerator{}, errors.New("missing or corrupted snapshot journal")
-	}
-	r := rlp.NewStream(bytes.NewReader(journal), 0)
-
-	// Read the snapshot generation progress for the disk layer
-	var generator journalGenerator
-	if err := r.Decode(&generator); err != nil {
-		return nil, journalGenerator{}, fmt.Errorf("failed to load snapshot progress marker: %v", err)
-	}
-	// Load all the snapshot diffs from the journal
-	snapshot, err := loadDiffLayer(base, r)
-	if err != nil {
-		return nil, generator, err
-	}
-	return snapshot, generator, nil
-}
-
 // loadAndParseJournal tries to parse the snapshot journal in latest format.
 func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) {
 	// Retrieve the disk layer generator. It must exist, no matter the
@@ -147,12 +126,17 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou
 }
 
 // loadSnapshot loads a pre-existing state snapshot backed by a key-value store.
-func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, error) {
+func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) {
+	// If snapshotting is disabled (initial sync in progress), don't do anything,
+	// wait for the chain to permit us to do something meaningful
+	if rawdb.ReadSnapshotDisabled(diskdb) {
+		return nil, true, nil
+	}
 	// Retrieve the block number and hash of the snapshot, failing if no snapshot
 	// is present in the database (or crashed mid-update).
 	baseRoot := rawdb.ReadSnapshotRoot(diskdb)
 	if baseRoot == (common.Hash{}) {
-		return nil, errors.New("missing or corrupted snapshot")
+		return nil, false, errors.New("missing or corrupted snapshot")
 	}
 	base := &diskLayer{
 		diskdb: diskdb,
@@ -160,15 +144,10 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 		cache:  fastcache.New(cache * 1024 * 1024),
 		root:   baseRoot,
 	}
-	var legacy bool
 	snapshot, generator, err := loadAndParseJournal(diskdb, base)
 	if err != nil {
 		log.Warn("Failed to load new-format journal", "error", err)
-		snapshot, generator, err = loadAndParseLegacyJournal(diskdb, base)
-		legacy = true
-	}
-	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	// Entire snapshot journal loaded, sanity check the head. If the loaded
 	// snapshot is not matched with current state root, print a warning log
@@ -182,8 +161,8 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 		// If it's legacy snapshot, or it's new-format snapshot but
 		// it's not in recovery mode, returns the error here for
 		// rebuilding the entire snapshot forcibly.
-		if legacy || !recovery {
-			return nil, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root)
+		if !recovery {
+			return nil, false, fmt.Errorf("head doesn't match snapshot: have %#x, want %#x", head, root)
 		}
 		// It's in snapshot recovery, the assumption is held that
 		// the disk layer is always higher than chain head. It can
@@ -193,14 +172,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 	}
 	// Everything loaded correctly, resume any suspended operations
 	if !generator.Done {
-		// If the generator was still wiping, restart one from scratch (fine for
-		// now as it's rare and the wiper deletes the stuff it touches anyway, so
-		// restarting won't incur a lot of extra database hops.
-		var wiper chan struct{}
-		if generator.Wiping {
-			log.Info("Resuming previous snapshot wipe")
-			wiper = wipeSnapshot(diskdb, false)
-		}
 		// Whether or not wiping was in progress, load any generator progress too
 		base.genMarker = generator.Marker
 		if base.genMarker == nil {
@@ -214,7 +185,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 			origin = binary.BigEndian.Uint64(generator.Marker)
 		}
 		go base.generate(&generatorStats{
-			wiping:   wiper,
 			origin:   origin,
 			start:    time.Now(),
 			accounts: generator.Accounts,
@@ -222,7 +192,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int,
 			storage:  common.StorageSize(generator.Storage),
 		})
 	}
-	return snapshot, nil
+	return snapshot, false, nil
 }
 
 // loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new
@@ -352,95 +322,3 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
 	log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root())
 	return base, nil
 }
-
-// LegacyJournal writes the persistent layer generator stats into a buffer
-// to be stored in the database as the snapshot journal.
-//
-// Note it's the legacy version which is only used in testing right now.
-func (dl *diskLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) {
-	// If the snapshot is currently being generated, abort it
-	var stats *generatorStats
-	if dl.genAbort != nil {
-		abort := make(chan *generatorStats)
-		dl.genAbort <- abort
-
-		if stats = <-abort; stats != nil {
-			stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker)
-		}
-	}
-	// Ensure the layer didn't get stale
-	dl.lock.RLock()
-	defer dl.lock.RUnlock()
-
-	if dl.stale {
-		return common.Hash{}, ErrSnapshotStale
-	}
-	// Write out the generator marker
-	entry := journalGenerator{
-		Done:   dl.genMarker == nil,
-		Marker: dl.genMarker,
-	}
-	if stats != nil {
-		entry.Wiping = (stats.wiping != nil)
-		entry.Accounts = stats.accounts
-		entry.Slots = stats.slots
-		entry.Storage = uint64(stats.storage)
-	}
-	log.Debug("Legacy journalled disk layer", "root", dl.root)
-	if err := rlp.Encode(buffer, entry); err != nil {
-		return common.Hash{}, err
-	}
-	return dl.root, nil
-}
-
-// Journal writes the memory layer contents into a buffer to be stored in the
-// database as the snapshot journal.
-//
-// Note it's the legacy version which is only used in testing right now.
-func (dl *diffLayer) LegacyJournal(buffer *bytes.Buffer) (common.Hash, error) {
-	// Journal the parent first
-	base, err := dl.parent.LegacyJournal(buffer)
-	if err != nil {
-		return common.Hash{}, err
-	}
-	// Ensure the layer didn't get stale
-	dl.lock.RLock()
-	defer dl.lock.RUnlock()
-
-	if dl.Stale() {
-		return common.Hash{}, ErrSnapshotStale
-	}
-	// Everything below was journalled, persist this layer too
-	if err := rlp.Encode(buffer, dl.root); err != nil {
-		return common.Hash{}, err
-	}
-	destructs := make([]journalDestruct, 0, len(dl.destructSet))
-	for hash := range dl.destructSet {
-		destructs = append(destructs, journalDestruct{Hash: hash})
-	}
-	if err := rlp.Encode(buffer, destructs); err != nil {
-		return common.Hash{}, err
-	}
-	accounts := make([]journalAccount, 0, len(dl.accountData))
-	for hash, blob := range dl.accountData {
-		accounts = append(accounts, journalAccount{Hash: hash, Blob: blob})
-	}
-	if err := rlp.Encode(buffer, accounts); err != nil {
-		return common.Hash{}, err
-	}
-	storage := make([]journalStorage, 0, len(dl.storageData))
-	for hash, slots := range dl.storageData {
-		keys := make([]common.Hash, 0, len(slots))
-		vals := make([][]byte, 0, len(slots))
-		for key, val := range slots {
-			keys = append(keys, key)
-			vals = append(vals, val)
-		}
-		storage = append(storage, journalStorage{Hash: hash, Keys: keys, Vals: vals})
-	}
-	if err := rlp.Encode(buffer, storage); err != nil {
-		return common.Hash{}, err
-	}
-	log.Debug("Legacy journalled diff layer", "root", dl.root, "parent", dl.parent.Root())
-	return base, nil
-}

core/state/snapshot/snapshot.go

@@ -137,10 +137,6 @@ type snapshot interface {
 	// flattening everything down (bad for reorgs).
 	Journal(buffer *bytes.Buffer) (common.Hash, error)
 
-	// LegacyJournal is basically identical to Journal. it's the legacy version for
-	// flushing legacy journal. Now the only purpose of this function is for testing.
-	LegacyJournal(buffer *bytes.Buffer) (common.Hash, error)
-
 	// Stale return whether this layer has become stale (was flattened across) or
 	// if it's still live.
 	Stale() bool
@@ -152,11 +148,11 @@ type snapshot interface {
 	StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool)
 }
 
-// SnapshotTree is an Ethereum state snapshot tree. It consists of one persistent
-// base layer backed by a key-value store, on top of which arbitrarily many in-
-// memory diff layers are topped. The memory diffs can form a tree with branching,
-// but the disk layer is singleton and common to all. If a reorg goes deeper than
-// the disk layer, everything needs to be deleted.
+// Tree is an Ethereum state snapshot tree. It consists of one persistent base
+// layer backed by a key-value store, on top of which arbitrarily many in-memory
+// diff layers are topped. The memory diffs can form a tree with branching, but
+// the disk layer is singleton and common to all. If a reorg goes deeper than the
+// disk layer, everything needs to be deleted.
 //
 // The goal of a state snapshot is twofold: to allow direct access to account and
 // storage data to avoid expensive multi-level trie lookups; and to allow sorted,
@@ -190,7 +186,11 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm
 		defer snap.waitBuild()
 	}
 	// Attempt to load a previously persisted snapshot and rebuild one if failed
-	head, err := loadSnapshot(diskdb, triedb, cache, root, recovery)
+	head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery)
+	if disabled {
+		log.Warn("Snapshot maintenance disabled (syncing)")
+		return snap, nil
+	}
 	if err != nil {
 		if rebuild {
 			log.Warn("Failed to load snapshot, regenerating", "err", err)
@@ -228,6 +228,55 @@ func (t *Tree) waitBuild() {
 	}
 }
 
+// Disable interrupts any pending snapshot generator, deletes all the snapshot
+// layers in memory and marks snapshots disabled globally. In order to resume
+// the snapshot functionality, the caller must invoke Rebuild.
+func (t *Tree) Disable() {
+	// Interrupt any live snapshot layers
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	for _, layer := range t.layers {
+		switch layer := layer.(type) {
+		case *diskLayer:
+			// If the base layer is generating, abort it
+			if layer.genAbort != nil {
+				abort := make(chan *generatorStats)
+				layer.genAbort <- abort
+				<-abort
+			}
+			// Layer should be inactive now, mark it as stale
+			layer.lock.Lock()
+			layer.stale = true
+			layer.lock.Unlock()
+
+		case *diffLayer:
+			// If the layer is a simple diff, simply mark as stale
+			layer.lock.Lock()
+			atomic.StoreUint32(&layer.stale, 1)
+			layer.lock.Unlock()
+
+		default:
+			panic(fmt.Sprintf("unknown layer type: %T", layer))
+		}
+	}
+	t.layers = map[common.Hash]snapshot{}
+
+	// Delete all snapshot liveness information from the database
+	batch := t.diskdb.NewBatch()
+
+	rawdb.WriteSnapshotDisabled(batch)
+	rawdb.DeleteSnapshotRoot(batch)
+	rawdb.DeleteSnapshotJournal(batch)
+	rawdb.DeleteSnapshotGenerator(batch)
+	rawdb.DeleteSnapshotRecoveryNumber(batch)
+	// Note, we don't delete the sync progress
+
+	if err := batch.Write(); err != nil {
+		log.Crit("Failed to disable snapshots", "err", err)
+	}
+}
+
 // Snapshot retrieves a snapshot belonging to the given block root, or nil if no
 // snapshot is maintained for that block.
 func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot {
@@ -283,11 +332,11 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m
 		return errSnapshotCycle
 	}
 	// Generate a new snapshot on top of the parent
-	parent := t.Snapshot(parentRoot).(snapshot)
+	parent := t.Snapshot(parentRoot)
 	if parent == nil {
 		return fmt.Errorf("parent [%#x] snapshot missing", parentRoot)
 	}
-	snap := parent.Update(blockRoot, destructs, accounts, storage)
+	snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage)
 
 	// Save the new snapshot for later
 	t.lock.Lock()
@@ -484,8 +533,17 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 			if key := it.Key(); len(key) == 65 { // TODO(karalabe): Yuck, we should move this into the iterator
 				batch.Delete(key)
 				base.cache.Del(key[1:])
-
 				snapshotFlushStorageItemMeter.Mark(1)
+
+				// Ensure we don't delete too much data blindly (contract can be
+				// huge). It's ok to flush, the root will go missing in case of a
+				// crash and we'll detect and regenerate the snapshot.
+				if batch.ValueSize() > ethdb.IdealBatchSize {
+					if err := batch.Write(); err != nil {
+						log.Crit("Failed to write storage deletions", "err", err)
+					}
+					batch.Reset()
+				}
 			}
 		}
 		it.Release()
@@ -503,6 +561,16 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 
 		snapshotFlushAccountItemMeter.Mark(1)
 		snapshotFlushAccountSizeMeter.Mark(int64(len(data)))
+
+		// Ensure we don't write too much data blindly. It's ok to flush, the
+		// root will go missing in case of a crash and we'll detect and regen
+		// the snapshot.
+		if batch.ValueSize() > ethdb.IdealBatchSize {
+			if err := batch.Write(); err != nil {
+				log.Crit("Failed to write storage deletions", "err", err)
+			}
+			batch.Reset()
+		}
 	}
 	// Push all the storage slots into the database
 	for accountHash, storage := range bottom.storageData {
@@ -603,29 +671,6 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) {
 	return base, nil
 }
 
-// LegacyJournal is basically identical to Journal. it's the legacy
-// version for flushing legacy journal. Now the only purpose of this
-// function is for testing.
-func (t *Tree) LegacyJournal(root common.Hash) (common.Hash, error) {
-	// Retrieve the head snapshot to journal from var snap snapshot
-	snap := t.Snapshot(root)
-	if snap == nil {
-		return common.Hash{}, fmt.Errorf("snapshot [%#x] missing", root)
-	}
-	// Run the journaling
-	t.lock.Lock()
-	defer t.lock.Unlock()
-
-	journal := new(bytes.Buffer)
-	base, err := snap.(snapshot).LegacyJournal(journal)
-	if err != nil {
-		return common.Hash{}, err
-	}
-	// Store the journal into the database and return
-	rawdb.WriteSnapshotJournal(t.diskdb, journal.Bytes())
-	return base, nil
-}
-
 // Rebuild wipes all available snapshot data from the persistent database and
 // discard all caches and diff layers. Afterwards, it starts a new snapshot
 // generator with the given root hash.
@@ -634,11 +679,9 @@ func (t *Tree) Rebuild(root common.Hash) {
 	defer t.lock.Unlock()
 
 	// Firstly delete any recovery flag in the database. Because now we are
-	// building a brand new snapshot.
+	// building a brand new snapshot. Also reenable the snapshot feature.
 	rawdb.DeleteSnapshotRecoveryNumber(t.diskdb)
-
-	// Track whether there's a wipe currently running and keep it alive if so
-	var wiper chan struct{}
+	rawdb.DeleteSnapshotDisabled(t.diskdb)
 
 	// Iterate over and mark all layers stale
 	for _, layer := range t.layers {
@@ -648,10 +691,7 @@ func (t *Tree) Rebuild(root common.Hash) {
 			if layer.genAbort != nil {
 				abort := make(chan *generatorStats)
 				layer.genAbort <- abort
-
-				if stats := <-abort; stats != nil {
-					wiper = stats.wiping
-				}
+				<-abort
 			}
 			// Layer should be inactive now, mark it as stale
 			layer.lock.Lock()
@@ -672,7 +712,7 @@ func (t *Tree) Rebuild(root common.Hash) {
 	// generator will run a wiper first if there's not one running right now.
 	log.Info("Rebuilding state snapshot")
 	t.layers = map[common.Hash]snapshot{
-		root: generateSnapshot(t.diskdb, t.triedb, t.cache, root, wiper),
+		root: generateSnapshot(t.diskdb, t.triedb, t.cache, root),
 	}
 }
 

core/state/snapshot/wipe.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 )
 
 // wipeSnapshot starts a goroutine to iterate over the entire key-value database
@@ -53,10 +54,10 @@ func wipeSnapshot(db ethdb.KeyValueStore, full bool) chan struct{} {
 // removed in sync to avoid data races. After all is done, the snapshot range of
 // the database is compacted to free up unused data blocks.
 func wipeContent(db ethdb.KeyValueStore) error {
-	if err := wipeKeyRange(db, "accounts", rawdb.SnapshotAccountPrefix, len(rawdb.SnapshotAccountPrefix)+common.HashLength); err != nil {
+	if err := wipeKeyRange(db, "accounts", rawdb.SnapshotAccountPrefix, nil, nil, len(rawdb.SnapshotAccountPrefix)+common.HashLength, snapWipedAccountMeter, true); err != nil {
 		return err
 	}
-	if err := wipeKeyRange(db, "storage", rawdb.SnapshotStoragePrefix, len(rawdb.SnapshotStoragePrefix)+2*common.HashLength); err != nil {
+	if err := wipeKeyRange(db, "storage", rawdb.SnapshotStoragePrefix, nil, nil, len(rawdb.SnapshotStoragePrefix)+2*common.HashLength, snapWipedStorageMeter, true); err != nil {
 		return err
 	}
 	// Compact the snapshot section of the database to get rid of unused space
@@ -82,8 +83,11 @@ func wipeContent(db ethdb.KeyValueStore) error {
 }
 
 // wipeKeyRange deletes a range of keys from the database starting with prefix
-// and having a specific total key length.
-func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int) error {
+// and having a specific total key length. The start and limit is optional for
+// specifying a particular key range for deletion.
+//
+// Origin is included for wiping and limit is excluded if they are specified.
+func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, origin []byte, limit []byte, keylen int, meter metrics.Meter, report bool) error {
 	// Batch deletions together to avoid holding an iterator for too long
 	var (
 		batch = db.NewBatch()
@@ -92,7 +96,11 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 	// Iterate over the key-range and delete all of them
 	start, logged := time.Now(), time.Now()
 
-	it := db.NewIterator(prefix, nil)
+	it := db.NewIterator(prefix, origin)
+	var stop []byte
+	if limit != nil {
+		stop = append(prefix, limit...)
+	}
 	for it.Next() {
 		// Skip any keys with the correct prefix but wrong length (trie nodes)
 		key := it.Key()
@@ -102,6 +110,9 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 		if len(key) != keylen {
 			continue
 		}
+		if stop != nil && bytes.Compare(key, stop) >= 0 {
+			break
+		}
 		// Delete the key and periodically recreate the batch and iterator
 		batch.Delete(key)
 		items++
@@ -116,7 +127,7 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 			seekPos := key[len(prefix):]
 			it = db.NewIterator(prefix, seekPos)
 
-			if time.Since(logged) > 8*time.Second {
+			if time.Since(logged) > 8*time.Second && report {
 				log.Info("Deleting state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start)))
 				logged = time.Now()
 			}
@@ -126,6 +137,11 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 	if err := batch.Write(); err != nil {
 		return err
 	}
+	if meter != nil {
+		meter.Mark(int64(items))
+	}
+	if report {
 		log.Info("Deleted state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start)))
+	}
 	return nil
 }

core/state/state_test.go

@@ -27,8 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-var toAddr = common.BytesToAddress
-
 type stateTest struct {
 	db    ethdb.Database
 	state *StateDB
@@ -46,11 +44,11 @@ func TestDump(t *testing.T) {
 	s := &stateTest{db: db, state: sdb}
 
 	// generate a few entries
-	obj1 := s.state.GetOrNewStateObject(toAddr([]byte{0x01}))
+	obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01}))
 	obj1.AddBalance(big.NewInt(22))
-	obj2 := s.state.GetOrNewStateObject(toAddr([]byte{0x01, 0x02}))
+	obj2 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01, 0x02}))
 	obj2.SetCode(crypto.Keccak256Hash([]byte{3, 3, 3, 3, 3, 3, 3}), []byte{3, 3, 3, 3, 3, 3, 3})
-	obj3 := s.state.GetOrNewStateObject(toAddr([]byte{0x02}))
+	obj3 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x02}))
 	obj3.SetBalance(big.NewInt(44))
 
 	// write some of them to the trie
@@ -108,7 +106,7 @@ func TestNull(t *testing.T) {
 }
 
 func TestSnapshot(t *testing.T) {
-	stateobjaddr := toAddr([]byte("aa"))
+	stateobjaddr := common.BytesToAddress([]byte("aa"))
 	var storageaddr common.Hash
 	data1 := common.BytesToHash([]byte{42})
 	data2 := common.BytesToHash([]byte{43})
@@ -150,8 +148,8 @@ func TestSnapshotEmpty(t *testing.T) {
 func TestSnapshot2(t *testing.T) {
 	state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
 
-	stateobjaddr0 := toAddr([]byte("so0"))
-	stateobjaddr1 := toAddr([]byte("so1"))
+	stateobjaddr0 := common.BytesToAddress([]byte("so0"))
+	stateobjaddr1 := common.BytesToAddress([]byte("so1"))
 	var storageaddr common.Hash
 
 	data0 := common.BytesToHash([]byte{17})

core/state/statedb.go

@@ -948,7 +948,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 	// The onleaf func is called _serially_, so we can reuse the same account
 	// for unmarshalling every time.
 	var account Account
-	root, err := s.trie.Commit(func(path []byte, leaf []byte, parent common.Hash) error {
+	root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error {
 		if err := rlp.DecodeBytes(leaf, &account); err != nil {
 			return nil
 		}

core/state/statedb_test.go

@@ -672,7 +672,7 @@ func TestDeleteCreateRevert(t *testing.T) {
 	// Create an initial state with a single contract
 	state, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()), nil)
 
-	addr := toAddr([]byte("so"))
+	addr := common.BytesToAddress([]byte("so"))
 	state.SetBalance(addr, big.NewInt(1))
 
 	root, _ := state.Commit(false)
@@ -705,11 +705,11 @@ func TestMissingTrieNodes(t *testing.T) {
 	db := NewDatabase(memDb)
 	var root common.Hash
 	state, _ := New(common.Hash{}, db, nil)
-	addr := toAddr([]byte("so"))
+	addr := common.BytesToAddress([]byte("so"))
 	{
 		state.SetBalance(addr, big.NewInt(1))
 		state.SetCode(addr, []byte{1, 2, 3})
-		a2 := toAddr([]byte("another"))
+		a2 := common.BytesToAddress([]byte("another"))
 		state.SetBalance(a2, big.NewInt(100))
 		state.SetCode(a2, []byte{1, 2, 4})
 		root, _ = state.Commit(false)

core/state/sync.go

@@ -26,17 +26,31 @@ import (
 )
 
 // NewStateSync create a new state trie download scheduler.
-func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom) *trie.Sync {
+func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync {
+	// Register the storage slot callback if the external callback is specified.
+	var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error
+	if onLeaf != nil {
+		onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error {
+			return onLeaf(paths, leaf)
+		}
+	}
+	// Register the account callback to connect the state trie and the storage
+	// trie belongs to the contract.
 	var syncer *trie.Sync
-	callback := func(path []byte, leaf []byte, parent common.Hash) error {
+	onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error {
+		if onLeaf != nil {
+			if err := onLeaf(paths, leaf); err != nil {
+				return err
+			}
+		}
 		var obj Account
 		if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil {
 			return err
 		}
-		syncer.AddSubTrie(obj.Root, path, parent, nil)
-		syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent)
+		syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot)
+		syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent)
 		return nil
 	}
-	syncer = trie.NewSync(root, database, callback, bloom)
+	syncer = trie.NewSync(root, database, onAccount, bloom)
 	return syncer
 }

core/state/sync_test.go

@@ -133,7 +133,7 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error {
 // Tests that an empty state is not scheduled for syncing.
 func TestEmptyStateSync(t *testing.T) {
 	empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
-	sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()))
+	sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), trie.NewSyncBloom(1, memorydb.New()), nil)
 	if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 {
 		t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes)
 	}
@@ -170,7 +170,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
-	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
+	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil)
 
 	nodes, paths, codes := sched.Missing(count)
 	var (
@@ -249,7 +249,7 @@ func TestIterativeDelayedStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
-	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
+	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil)
 
 	nodes, _, codes := sched.Missing(0)
 	queue := append(append([]common.Hash{}, nodes...), codes...)
@@ -297,7 +297,7 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
-	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
+	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil)
 
 	queue := make(map[common.Hash]struct{})
 	nodes, _, codes := sched.Missing(count)
@@ -347,7 +347,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
-	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
+	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil)
 
 	queue := make(map[common.Hash]struct{})
 	nodes, _, codes := sched.Missing(0)
@@ -414,7 +414,7 @@ func TestIncompleteStateSync(t *testing.T) {
 
 	// Create a destination state and sync with the scheduler
 	dstDb := rawdb.NewMemoryDatabase()
-	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
+	sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb), nil)
 
 	var added []common.Hash
 

core/state_transition.go

@@ -262,10 +262,9 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
 	}
 
 	// Set up the initial access list.
-	if st.evm.ChainConfig().IsBerlin(st.evm.Context.BlockNumber) {
-		st.state.PrepareAccessList(msg.From(), msg.To(), st.evm.ActivePrecompiles(), msg.AccessList())
+	if rules := st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber); rules.IsBerlin {
+		st.state.PrepareAccessList(msg.From(), msg.To(), vm.ActivePrecompiles(rules), msg.AccessList())
 	}
-
 	var (
 		ret   []byte
 		vmerr error // vm errors do not effect consensus and are therefore not assigned to err

core/tx_pool.go

@@ -949,7 +949,7 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) {
 	}
 }
 
-// requestPromoteExecutables requests a pool reset to the new head block.
+// requestReset requests a pool reset to the new head block.
 // The returned channel is closed when the reset has occurred.
 func (pool *TxPool) requestReset(oldHead *types.Header, newHead *types.Header) chan struct{} {
 	select {