Merge branch 'ethereum:master' into master

.gitmodules

@@ -2,3 +2,7 @@
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
 	shallow = true
+[submodule "evm-benchmarks"]
+	path = tests/evm-benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks
+	shallow = true

.golangci.yml

@@ -1,7 +1,7 @@
 # This file configures github.com/golangci/golangci-lint.
 
 run:
-  timeout: 5m
+  timeout: 20m
   tests: true
   # default is true. Enables skipping of directories:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$

README.md

@@ -165,7 +165,7 @@ saving your blockchain as well as map the default ports. There is also an `alpin
 available for a slim version of the image.
 
 Do not forget `--http.addr 0.0.0.0`, if you want to access RPC from other containers
-and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not
+and/or hosts. By default, `geth` binds to the local interface and RPC endpoints are not
 accessible from the outside.
 
 ### Programmatically interfacing `geth` nodes

SECURITY.md

@@ -19,7 +19,7 @@ Audit reports are published in the `docs` folder: https://github.com/ethereum/go
 
 **Please do not file a public ticket** mentioning the vulnerability.
 
-To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publically disclosed security vulnerabilities.
+To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publicly disclosed security vulnerabilities.
 
 Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number.
 

accounts/abi/argument.go

@@ -81,13 +81,7 @@ func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) {
 		if len(arguments) != 0 {
 			return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
 		}
-		// Nothing to unmarshal, return default variables
+		return make([]interface{}, 0), nil
-		nonIndexedArgs := arguments.NonIndexed()
-		defaultVars := make([]interface{}, len(nonIndexedArgs))
-		for index, arg := range nonIndexedArgs {
-			defaultVars[index] = reflect.New(arg.Type.GetType())
-		}
-		return defaultVars, nil
 	}
 	return arguments.UnpackValues(data)
 }

accounts/abi/bind/backends/simulated.go

@@ -230,6 +230,9 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common
 	defer b.mu.Unlock()
 
 	receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash, b.config)
+	if receipt == nil {
+		return nil, ethereum.NotFound
+	}
 	return receipt, nil
 }
 
@@ -639,7 +642,6 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 }
 
 // SendTransaction updates the pending block to include the given transaction.
-// It panics if the transaction is invalid.
 func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transaction) error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -647,17 +649,17 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 	// Get the last block
 	block, err := b.blockByHash(ctx, b.pendingBlock.ParentHash())
 	if err != nil {
-		panic("could not fetch parent")
+		return fmt.Errorf("could not fetch parent")
 	}
 	// Check transaction validity
 	signer := types.MakeSigner(b.blockchain.Config(), block.Number())
 	sender, err := types.Sender(signer, tx)
 	if err != nil {
-		panic(fmt.Errorf("invalid transaction: %v", err))
+		return fmt.Errorf("invalid transaction: %v", err)
 	}
 	nonce := b.pendingState.GetNonce(sender)
 	if tx.Nonce() != nonce {
-		panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce))
+		return fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce)
 	}
 	// Include tx in chain
 	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {

accounts/abi/bind/backends/simulated_test.go

@@ -496,7 +496,7 @@ func TestEstimateGas(t *testing.T) {
 			GasPrice: big.NewInt(0),
 			Value:    nil,
 			Data:     common.Hex2Bytes("b9b046f9"),
-		}, 0, errors.New("invalid opcode: opcode 0xfe not defined"), nil},
+		}, 0, errors.New("invalid opcode: INVALID"), nil},
 
 		{"Valid", ethereum.CallMsg{
 			From:     addr,

accounts/abi/bind/util.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"time"
 
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -35,14 +36,16 @@ func WaitMined(ctx context.Context, b DeployBackend, tx *types.Transaction) (*ty
 	logger := log.New("hash", tx.Hash())
 	for {
 		receipt, err := b.TransactionReceipt(ctx, tx.Hash())
-		if receipt != nil {
+		if err == nil {
 			return receipt, nil
 		}
-		if err != nil {
+
-			logger.Trace("Receipt retrieval failed", "err", err)
+		if errors.Is(err, ethereum.NotFound) {
-		} else {
 			logger.Trace("Transaction not yet mined")
+		} else {
+			logger.Trace("Receipt retrieval failed", "err", err)
 		}
+
 		// Wait for the next round.
 		select {
 		case <-ctx.Done():

accounts/abi/unpack.go

@@ -290,7 +290,7 @@ func tuplePointsTo(index int, output []byte) (start int, err error) {
 	offset := big.NewInt(0).SetBytes(output[index : index+32])
 	outputLen := big.NewInt(int64(len(output)))
 
-	if offset.Cmp(big.NewInt(int64(len(output)))) > 0 {
+	if offset.Cmp(outputLen) > 0 {
 		return 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", offset, outputLen)
 	}
 	if offset.BitLen() > 63 {

accounts/accounts.go

@@ -46,7 +46,7 @@ const (
 // accounts (derived from the same seed).
 type Wallet interface {
 	// URL retrieves the canonical path under which this wallet is reachable. It is
-	// user by upper layers to define a sorting order over all wallets from multiple
+	// used by upper layers to define a sorting order over all wallets from multiple
 	// backends.
 	URL() URL
 
@@ -89,7 +89,7 @@ type Wallet interface {
 	// accounts.
 	//
 	// Note, self derivation will increment the last component of the specified path
-	// opposed to decending into a child path to allow discovering accounts starting
+	// opposed to descending into a child path to allow discovering accounts starting
 	// from non zero components.
 	//
 	// Some hardware wallets switched derivation paths through their evolution, so
@@ -105,7 +105,7 @@ type Wallet interface {
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
 	// If the wallet requires additional authentication to sign the request (e.g.
-	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// a password to decrypt the account, or a PIN code to verify the transaction),
 	// an AuthNeededError instance will be returned, containing infos for the user
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignDataWithPassphrase, or by other means (e.g. unlock
@@ -124,13 +124,13 @@ type Wallet interface {
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
 	// If the wallet requires additional authentication to sign the request (e.g.
-	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// a password to decrypt the account, or a PIN code to verify the transaction),
 	// an AuthNeededError instance will be returned, containing infos for the user
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignTextWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
 	//
-	// This method should return the signature in 'canonical' format, with v 0 or 1
+	// This method should return the signature in 'canonical' format, with v 0 or 1.
 	SignText(account Account, text []byte) ([]byte, error)
 
 	// SignTextWithPassphrase is identical to Signtext, but also takes a password
@@ -176,7 +176,7 @@ type Backend interface {
 // TextHash is a helper function that calculates a hash for the given message that can be
 // safely used to calculate a signature from.
 //
-// The hash is calulcated as
+// The hash is calculated as
 //   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
 //
 // This gives context to the signed message and prevents signing of transactions.
@@ -188,7 +188,7 @@ func TextHash(data []byte) []byte {
 // TextAndHash is a helper function that calculates a hash for the given message that can be
 // safely used to calculate a signature from.
 //
-// The hash is calulcated as
+// The hash is calculated as
 //   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
 //
 // This gives context to the signed message and prevents signing of transactions.

accounts/errors.go

@@ -42,7 +42,7 @@ var ErrInvalidPassphrase = errors.New("invalid password")
 var ErrWalletAlreadyOpen = errors.New("wallet already open")
 
 // ErrWalletClosed is returned if a wallet is attempted to be opened the
-// secodn time.
+// second time.
 var ErrWalletClosed = errors.New("wallet closed")
 
 // AuthNeededError is returned by backends for signing requests where the user

accounts/scwallet/wallet.go

@@ -638,7 +638,7 @@ func (w *Wallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Accoun
 // accounts.
 //
 // Note, self derivation will increment the last component of the specified path
-// opposed to decending into a child path to allow discovering accounts starting
+// opposed to descending into a child path to allow discovering accounts starting
 // from non zero components.
 //
 // Some hardware wallets switched derivation paths through their evolution, so

accounts/usbwallet/wallet.go

@@ -496,7 +496,7 @@ func (w *wallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Accoun
 // accounts.
 //
 // Note, self derivation will increment the last component of the specified path
-// opposed to decending into a child path to allow discovering accounts starting
+// opposed to descending into a child path to allow discovering accounts starting
 // from non zero components.
 //
 // Some hardware wallets switched derivation paths through their evolution, so

build/checksums.txt

@@ -1,19 +1,19 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-2255eb3e4e824dd7d5fcdc2e7f84534371c186312e546fb1086a34c17752f431  go1.17.2.src.tar.gz
+3defb9a09bed042403195e872dcbc8c6fae1485963332279668ec52e80a95a2d  go1.17.5.src.tar.gz
-7914497a302a132a465d33f5ee044ce05568bacdb390ab805cb75a3435a23f94  go1.17.2.darwin-amd64.tar.gz
+2db6a5d25815b56072465a2cacc8ed426c18f1d5fc26c1fc8c4f5a7188658264  go1.17.5.darwin-amd64.tar.gz
-ce8771bd3edfb5b28104084b56bbb532eeb47fbb7769c3e664c6223712c30904  go1.17.2.darwin-arm64.tar.gz
+111f71166de0cb8089bb3e8f9f5b02d76e1bf1309256824d4062a47b0e5f98e0  go1.17.5.darwin-arm64.tar.gz
-8cea5b8d1f8e8cbb58069bfed58954c71c5b1aca2f3c857765dae83bf724d0d7  go1.17.2.freebsd-386.tar.gz
+443c1cd9768df02085014f1eb034ebc7dbe032ffc8a9bb9f2e6617d037eee23c  go1.17.5.freebsd-386.tar.gz
-c96e57218fb03e74d683ad63b1684d44c89d5e5b994f36102b33dce21b58499a  go1.17.2.freebsd-amd64.tar.gz
+17180bdc4126acffd0ebf86d66ef5cbc3488b6734e93374fb00eb09494e006d3  go1.17.5.freebsd-amd64.tar.gz
-8617f2e40d51076983502894181ae639d1d8101bfbc4d7463a2b442f239f5596  go1.17.2.linux-386.tar.gz
+4f4914303bc18f24fd137a97e595735308f5ce81323c7224c12466fd763fc59f  go1.17.5.linux-386.tar.gz
-f242a9db6a0ad1846de7b6d94d507915d14062660616a61ef7c808a76e4f1676  go1.17.2.linux-amd64.tar.gz
+bd78114b0d441b029c8fe0341f4910370925a4d270a6a590668840675b0c653e  go1.17.5.linux-amd64.tar.gz
-a5a43c9cdabdb9f371d56951b14290eba8ce2f9b0db48fb5fc657943984fd4fc  go1.17.2.linux-arm64.tar.gz
+6f95ce3da40d9ce1355e48f31f4eb6508382415ca4d7413b1e7a3314e6430e7e  go1.17.5.linux-arm64.tar.gz
-04d16105008230a9763005be05606f7eb1c683a3dbf0fbfed4034b23889cb7f2  go1.17.2.linux-armv6l.tar.gz
+aa1fb6c53b4fe72f159333362a10aca37ae938bde8adc9c6eaf2a8e87d1e47de  go1.17.5.linux-armv6l.tar.gz
-12e2dc7e0ffeebe77083f267ef6705fec1621cdf2ed6489b3af04a13597ed68d  go1.17.2.linux-ppc64le.tar.gz
+3d4be616e568f0a02cb7f7769bcaafda4b0969ed0f9bb4277619930b96847e70  go1.17.5.linux-ppc64le.tar.gz
-c4b2349a8d11350ca038b8c57f3cc58dc0b31284bcbed4f7fca39aeed28b4a51  go1.17.2.linux-s390x.tar.gz
+8087d4fe991e82804e6485c26568c2e0ee0bfde00ceb9015dc86cb6bf84ef40b  go1.17.5.linux-s390x.tar.gz
-8a85257a351996fdf045fe95ed5fdd6917dd48636d562dd11dedf193005a53e0  go1.17.2.windows-386.zip
+6d7b9948ee14a906b14f5cbebdfab63cd6828b0b618160847ecd3cc3470a26fe  go1.17.5.windows-386.zip
-fa6da0b829a66f5fab7e4e312fd6aa1b2d8f045c7ecee83b3d00f6fe5306759a  go1.17.2.windows-amd64.zip
+671faf99cd5d81cd7e40936c0a94363c64d654faa0148d2af4bbc262555620b9  go1.17.5.windows-amd64.zip
-00575c85dc7a129ba892685a456b27a3f3670f71c8bfde1c5ad151f771d55df7  go1.17.2.windows-arm64.zip
+45e88676b68e9cf364be469b5a27965397f4e339aa622c2f52c10433c56e5030  go1.17.5.windows-arm64.zip
 
 d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa  golangci-lint-1.42.0-darwin-amd64.tar.gz
 e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea  golangci-lint-1.42.0-darwin-arm64.tar.gz

build/ci.go

@@ -147,7 +147,7 @@ var (
 	// This is the version of go that will be downloaded by
 	//
 	//     go run ci.go install -dlgo
-	dlgoVersion = "1.17.2"
+	dlgoVersion = "1.17.5"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -334,7 +334,11 @@ func downloadLinter(cachedir string) string {
 	const version = "1.42.0"
 
 	csdb := build.MustLoadChecksums("build/checksums.txt")
-	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)
+	arch := runtime.GOARCH
+	if arch == "arm" {
+		arch += "v" + os.Getenv("GOARM")
+	}
+	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
 	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base)
 	archivePath := filepath.Join(cachedir, base+".tar.gz")
 	if err := csdb.DownloadFile(url, archivePath); err != nil {

cmd/ethkey/utils.go

@@ -49,7 +49,7 @@ func getPassphrase(ctx *cli.Context, confirmation bool) string {
 // signHash is a helper function that calculates a hash for the given message
 // that can be safely used to calculate a signature from.
 //
-// The hash is calulcated as
+// The hash is calculated as
 //   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
 //
 // This gives context to the signed message and prevents signing of transactions.

cmd/evm/internal/t8ntool/execution.go

@@ -67,6 +67,7 @@ type ommer struct {
 type stEnv struct {
 	Coinbase         common.Address                      `json:"currentCoinbase"   gencodec:"required"`
 	Difficulty       *big.Int                            `json:"currentDifficulty"`
+	Random           *big.Int                            `json:"currentRandom"`
 	ParentDifficulty *big.Int                            `json:"parentDifficulty"`
 	GasLimit         uint64                              `json:"currentGasLimit"   gencodec:"required"`
 	Number           uint64                              `json:"currentNumber"     gencodec:"required"`
@@ -81,6 +82,7 @@ type stEnv struct {
 type stEnvMarshaling struct {
 	Coinbase         common.UnprefixedAddress
 	Difficulty       *math.HexOrDecimal256
+	Random           *math.HexOrDecimal256
 	ParentDifficulty *math.HexOrDecimal256
 	GasLimit         math.HexOrDecimal64
 	Number           math.HexOrDecimal64
@@ -139,6 +141,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	if pre.Env.BaseFee != nil {
 		vmContext.BaseFee = new(big.Int).Set(pre.Env.BaseFee)
 	}
+	// If random is defined, add it to the vmContext.
+	if pre.Env.Random != nil {
+		rnd := common.BigToHash(pre.Env.Random)
+		vmContext.Random = &rnd
+	}
 	// If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's
 	// done in StateProcessor.Process(block, ...), right before transactions are applied.
 	if chainConfig.DAOForkSupport &&

cmd/evm/internal/t8ntool/gen_stenv.go

@@ -18,6 +18,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 	type stEnv struct {
 		Coinbase         common.UnprefixedAddress            `json:"currentCoinbase"   gencodec:"required"`
 		Difficulty       *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Random           *math.HexOrDecimal256               `json:"currentRandom"`
 		ParentDifficulty *math.HexOrDecimal256               `json:"parentDifficulty"`
 		GasLimit         math.HexOrDecimal64                 `json:"currentGasLimit"   gencodec:"required"`
 		Number           math.HexOrDecimal64                 `json:"currentNumber"     gencodec:"required"`
@@ -31,6 +32,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 	var enc stEnv
 	enc.Coinbase = common.UnprefixedAddress(s.Coinbase)
 	enc.Difficulty = (*math.HexOrDecimal256)(s.Difficulty)
+	enc.Random = (*math.HexOrDecimal256)(s.Random)
 	enc.ParentDifficulty = (*math.HexOrDecimal256)(s.ParentDifficulty)
 	enc.GasLimit = math.HexOrDecimal64(s.GasLimit)
 	enc.Number = math.HexOrDecimal64(s.Number)
@@ -48,6 +50,7 @@ func (s *stEnv) UnmarshalJSON(input []byte) error {
 	type stEnv struct {
 		Coinbase         *common.UnprefixedAddress           `json:"currentCoinbase"   gencodec:"required"`
 		Difficulty       *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Random           *math.HexOrDecimal256               `json:"currentRandom"`
 		ParentDifficulty *math.HexOrDecimal256               `json:"parentDifficulty"`
 		GasLimit         *math.HexOrDecimal64                `json:"currentGasLimit"   gencodec:"required"`
 		Number           *math.HexOrDecimal64                `json:"currentNumber"     gencodec:"required"`
@@ -69,6 +72,9 @@ func (s *stEnv) UnmarshalJSON(input []byte) error {
 	if dec.Difficulty != nil {
 		s.Difficulty = (*big.Int)(dec.Difficulty)
 	}
+	if dec.Random != nil {
+		s.Random = (*big.Int)(dec.Random)
+	}
 	if dec.ParentDifficulty != nil {
 		s.ParentDifficulty = (*big.Int)(dec.ParentDifficulty)
 	}

cmd/evm/internal/t8ntool/transition.go

@@ -252,6 +252,10 @@ func Transition(ctx *cli.Context) error {
 			return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
 		}
 	}
+	// Sanity check, to not `panic` in state_transition
+	if prestate.Env.Random != nil && !chainConfig.IsLondon(big.NewInt(int64(prestate.Env.Number))) {
+		return NewError(ErrorConfig, errors.New("can only apply RANDOM on top of London chainrules"))
+	}
 	if env := prestate.Env; env.Difficulty == nil {
 		// If difficulty was not provided by caller, we need to calculate it.
 		switch {

cmd/geth/accountcmd_test.go

@@ -120,7 +120,7 @@ func importAccountWithExpect(t *testing.T, key string, expected string) {
 	if err := ioutil.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil {
 		t.Error(err)
 	}
-	geth := runGeth(t, "account", "import", keyfile, "-password", passwordFile)
+	geth := runGeth(t, "--lightkdf", "account", "import", keyfile, "-password", passwordFile)
 	defer geth.ExpectExit()
 	geth.Expect(expected)
 }

cmd/geth/config.go

@@ -161,7 +161,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) {
 		cfg.Eth.OverrideTerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name))
 	}
-	backend, _ := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name))
+	backend, _ := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {

cmd/geth/dbcmd.go

@@ -34,9 +34,11 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/olekukonko/tablewriter"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -69,6 +71,7 @@ Remove blockchain and state databases`,
 			dbDumpFreezerIndex,
 			dbImportCmd,
 			dbExportCmd,
+			dbMetadataCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
@@ -233,6 +236,21 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		},
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
+	dbMetadataCmd = cli.Command{
+		Action: utils.MigrateFlags(showMetaData),
+		Name:   "metadata",
+		Usage:  "Shows metadata about the chain status.",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.RopstenFlag,
+			utils.SepoliaFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+		},
+		Description: "Shows metadata about the chain status.",
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -539,7 +557,7 @@ func freezerInspect(ctx *cli.Context) error {
 	defer stack.Close()
 	path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
 	log.Info("Opening freezer", "location", path, "name", kind)
-	if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy); err != nil {
+	if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil {
 		return err
 	} else {
 		f.DumpIndex(start, end)
@@ -685,3 +703,50 @@ func exportChaindata(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
+
+func showMetaData(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	ancients, err := db.Ancients()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)
+	}
+	pp := func(val *uint64) string {
+		if val == nil {
+			return "<nil>"
+		}
+		return fmt.Sprintf("%d (0x%x)", *val, *val)
+	}
+	data := [][]string{
+		{"databaseVersion", pp(rawdb.ReadDatabaseVersion(db))},
+		{"headBlockHash", fmt.Sprintf("%v", rawdb.ReadHeadBlockHash(db))},
+		{"headFastBlockHash", fmt.Sprintf("%v", rawdb.ReadHeadFastBlockHash(db))},
+		{"headHeaderHash", fmt.Sprintf("%v", rawdb.ReadHeadHeaderHash(db))}}
+	if b := rawdb.ReadHeadBlock(db); b != nil {
+		data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())})
+		data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())})
+		data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())})
+	}
+	if h := rawdb.ReadHeadHeader(db); h != nil {
+		data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())})
+		data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)})
+		data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (0x%x)", h.Number, h.Number)})
+	}
+	data = append(data, [][]string{{"frozen", fmt.Sprintf("%d items", ancients)},
+		{"lastPivotNumber", pp(rawdb.ReadLastPivotNumber(db))},
+		{"len(snapshotSyncStatus)", fmt.Sprintf("%d bytes", len(rawdb.ReadSnapshotSyncStatus(db)))},
+		{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))},
+		{"snapshotDisabled", fmt.Sprintf("%v", rawdb.ReadSnapshotDisabled(db))},
+		{"snapshotJournal", fmt.Sprintf("%d bytes", len(rawdb.ReadSnapshotJournal(db)))},
+		{"snapshotRecoveryNumber", pp(rawdb.ReadSnapshotRecoveryNumber(db))},
+		{"snapshotRoot", fmt.Sprintf("%v", rawdb.ReadSnapshotRoot(db))},
+		{"txIndexTail", pp(rawdb.ReadTxIndexTail(db))},
+		{"fastTxLookupLimit", pp(rawdb.ReadFastTxLookupLimit(db))},
+	}...)
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Field", "Value"})
+	table.AppendBulk(data)
+	table.Render()
+	return nil
+}

cmd/geth/main.go

@@ -157,7 +157,6 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
-		utils.CatalystFlag,
 	}
 
 	rpcFlags = []cli.Flag{
@@ -208,7 +207,7 @@ func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2021 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2022 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
@@ -274,6 +273,9 @@ func prepare(ctx *cli.Context) {
 	case ctx.GlobalIsSet(utils.RopstenFlag.Name):
 		log.Info("Starting Geth on Ropsten testnet...")
 
+	case ctx.GlobalIsSet(utils.SepoliaFlag.Name):
+		log.Info("Starting Geth on Sepolia testnet...")
+
 	case ctx.GlobalIsSet(utils.RinkebyFlag.Name):
 		log.Info("Starting Geth on Rinkeby testnet...")
 
@@ -289,7 +291,11 @@ func prepare(ctx *cli.Context) {
 	// If we're a full node on mainnet without --cache specified, bump default cache allowance
 	if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) {
 		// Make sure we're not on any supported preconfigured testnet either
-		if !ctx.GlobalIsSet(utils.RopstenFlag.Name) && !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && !ctx.GlobalIsSet(utils.GoerliFlag.Name) && !ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
+		if !ctx.GlobalIsSet(utils.RopstenFlag.Name) &&
+			!ctx.GlobalIsSet(utils.SepoliaFlag.Name) &&
+			!ctx.GlobalIsSet(utils.RinkebyFlag.Name) &&
+			!ctx.GlobalIsSet(utils.GoerliFlag.Name) &&
+			!ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
 			// Nope, we're really on mainnet. Bump that cache up!
 			log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)
 			ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096))

cmd/geth/snapshot.go

@@ -418,8 +418,7 @@ func traverseRawState(ctx *cli.Context) error {
 					// Check the present for non-empty hash node(embedded node doesn't
 					// have their own hash).
 					if node != (common.Hash{}) {
-						blob := rawdb.ReadTrieNode(chaindb, node)
+						if !rawdb.HasTrieNode(chaindb, node) {
-						if len(blob) == 0 {
 							log.Error("Missing trie node(storage)", "hash", node)
 							return errors.New("missing storage")
 						}

cmd/geth/usage.go

@@ -229,7 +229,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.SnapshotFlag,
 			utils.BloomFilterSizeFlag,
 			cli.HelpFlag,
-			utils.CatalystFlag,
 		},
 	},
 }

cmd/geth/version_check_test.go

@@ -25,6 +25,8 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+
+	"github.com/jedisct1/go-minisign"
 )
 
 func TestVerification(t *testing.T) {
@@ -128,3 +130,39 @@ func TestMatching(t *testing.T) {
 		}
 	}
 }
+
+func TestGethPubKeysParseable(t *testing.T) {
+	for _, pubkey := range gethPubKeys {
+		_, err := minisign.NewPublicKey(pubkey)
+		if err != nil {
+			t.Errorf("Should be parseable")
+		}
+	}
+}
+
+func TestKeyID(t *testing.T) {
+	type args struct {
+		id [8]byte
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"@holiman key", args{id: extractKeyId(gethPubKeys[0])}, "FB1D084D39BAEC24"},
+		{"second key", args{id: extractKeyId(gethPubKeys[1])}, "138B1CA303E51687"},
+		{"third key", args{id: extractKeyId(gethPubKeys[2])}, "FD9813B2D2098484"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := keyID(tt.args.id); got != tt.want {
+				t.Errorf("keyID() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func extractKeyId(pubkey string) [8]byte {
+	p, _ := minisign.NewPublicKey(pubkey)
+	return p.KeyId
+}

cmd/utils/flags.go

@@ -45,7 +45,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/eth/catalyst"
+	ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/gasprice"
@@ -56,6 +56,7 @@ import (
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/les"
+	lescatalyst "github.com/ethereum/go-ethereum/les/catalyst"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/metrics/exp"
@@ -789,11 +790,6 @@ var (
 		Usage: "InfluxDB organization name (v2 only)",
 		Value: metrics.DefaultConfig.InfluxDBOrganization,
 	}
-
-	CatalystFlag = cli.BoolFlag{
-		Name:  "catalyst",
-		Usage: "Catalyst mode (eth2 integration testing)",
-	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1673,9 +1669,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// Create a new developer genesis block or reuse existing one
 		cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), ctx.GlobalUint64(DeveloperGasLimitFlag.Name), developer.Address)
 		if ctx.GlobalIsSet(DataDirFlag.Name) {
+			// If datadir doesn't exist we need to open db in write-mode
+			// so leveldb can create files.
+			readonly := true
+			if !common.FileExist(stack.ResolvePath("chaindata")) {
+				readonly = false
+			}
 			// Check if we have an already initialized chain and fall back to
 			// that if so. Otherwise we need to generate a new genesis spec.
-			chaindb := MakeChainDatabase(ctx, stack, false) // TODO (MariusVanDerWijden) make this read only
+			chaindb := MakeChainDatabase(ctx, stack, readonly)
 			if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
 				cfg.Genesis = nil // fallback to db content
 			}
@@ -1710,15 +1712,15 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) {
 // RegisterEthService adds an Ethereum client to the stack.
 // The second return value is the full node instance, which may be nil if the
 // node is running as a light client.
-func RegisterEthService(stack *node.Node, cfg *ethconfig.Config, isCatalyst bool) (ethapi.Backend, *eth.Ethereum) {
+func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend, *eth.Ethereum) {
 	if cfg.SyncMode == downloader.LightSync {
 		backend, err := les.New(stack, cfg)
 		if err != nil {
 			Fatalf("Failed to register the Ethereum service: %v", err)
 		}
 		stack.RegisterAPIs(tracers.APIs(backend.ApiBackend))
-		if isCatalyst {
+		if backend.BlockChain().Config().TerminalTotalDifficulty != nil {
-			if err := catalyst.RegisterLight(stack, backend); err != nil {
+			if err := lescatalyst.Register(stack, backend); err != nil {
 				Fatalf("Failed to register the catalyst service: %v", err)
 			}
 		}
@@ -1734,8 +1736,8 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config, isCatalyst bool
 			Fatalf("Failed to create the LES server: %v", err)
 		}
 	}
-	if isCatalyst {
+	if backend.BlockChain().Config().TerminalTotalDifficulty != nil {
-		if err := catalyst.Register(stack, backend); err != nil {
+		if err := ethcatalyst.Register(stack, backend); err != nil {
 			Fatalf("Failed to register the catalyst service: %v", err)
 		}
 	}

consensus/beacon/consensus.go

@@ -43,7 +43,6 @@ var (
 // error types into the consensus package.
 var (
 	errTooManyUncles    = errors.New("too many uncles")
-	errInvalidMixDigest = errors.New("invalid mix digest")
 	errInvalidNonce     = errors.New("invalid nonce")
 	errInvalidUncleHash = errors.New("invalid uncle hash")
 )
@@ -182,10 +181,7 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa
 	if len(header.Extra) > 32 {
 		return fmt.Errorf("extra-data longer than 32 bytes (%d)", len(header.Extra))
 	}
-	// Verify the seal parts. Ensure the mixhash, nonce and uncle hash are the expected value.
+	// Verify the seal parts. Ensure the nonce and uncle hash are the expected value.
-	if header.MixDigest != (common.Hash{}) {
-		return errInvalidMixDigest
-	}
 	if header.Nonce != beaconNonce {
 		return errInvalidNonce
 	}

core/asm/lexer.go

@@ -68,10 +68,10 @@ func (it tokenType) String() string {
 
 var stringtokenTypes = []string{
 	eof:              "EOF",
+	lineStart:        "new line",
+	lineEnd:          "end of line",
 	invalidStatement: "invalid statement",
 	element:          "element",
-	lineEnd:          "end of line",
-	lineStart:        "new line",
 	label:            "label",
 	labelDef:         "label definition",
 	number:           "number",

core/beacon/errors.go

@@ -0,0 +1,29 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+package beacon
+
+import "github.com/ethereum/go-ethereum/rpc"
+
+var (
+	VALID              = GenericStringResponse{"VALID"}
+	SUCCESS            = GenericStringResponse{"SUCCESS"}
+	INVALID            = ForkChoiceResponse{Status: "INVALID", PayloadID: nil}
+	SYNCING            = ForkChoiceResponse{Status: "SYNCING", PayloadID: nil}
+	GenericServerError = rpc.CustomError{Code: -32000, ValidationError: "Server error"}
+	UnknownPayload     = rpc.CustomError{Code: -32001, ValidationError: "Unknown payload"}
+	InvalidTB          = rpc.CustomError{Code: -32002, ValidationError: "Invalid terminal block"}
+)

core/beacon/gen_blockparams.go

@@ -1,6 +1,6 @@
 // Code generated by github.com/fjl/gencodec. DO NOT EDIT.
 
-package catalyst
+package beacon
 
 import (
 	"encoding/json"
@@ -15,23 +15,23 @@ var _ = (*payloadAttributesMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (p PayloadAttributesV1) MarshalJSON() ([]byte, error) {
 	type PayloadAttributesV1 struct {
-		Timestamp    hexutil.Uint64 `json:"timestamp"     gencodec:"required"`
+		Timestamp             hexutil.Uint64 `json:"timestamp"     gencodec:"required"`
-		Random       common.Hash    `json:"random"        gencodec:"required"`
+		Random                common.Hash    `json:"random"        gencodec:"required"`
-		FeeRecipient common.Address `json:"feeRecipient"  gencodec:"required"`
+		SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient"  gencodec:"required"`
 	}
 	var enc PayloadAttributesV1
 	enc.Timestamp = hexutil.Uint64(p.Timestamp)
 	enc.Random = p.Random
-	enc.FeeRecipient = p.FeeRecipient
+	enc.SuggestedFeeRecipient = p.SuggestedFeeRecipient
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (p *PayloadAttributesV1) UnmarshalJSON(input []byte) error {
 	type PayloadAttributesV1 struct {
-		Timestamp    *hexutil.Uint64 `json:"timestamp"     gencodec:"required"`
+		Timestamp             *hexutil.Uint64 `json:"timestamp"     gencodec:"required"`
-		Random       *common.Hash    `json:"random"        gencodec:"required"`
+		Random                *common.Hash    `json:"random"        gencodec:"required"`
-		FeeRecipient *common.Address `json:"feeRecipient"  gencodec:"required"`
+		SuggestedFeeRecipient *common.Address `json:"suggestedFeeRecipient"  gencodec:"required"`
 	}
 	var dec PayloadAttributesV1
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -45,9 +45,9 @@ func (p *PayloadAttributesV1) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'random' for PayloadAttributesV1")
 	}
 	p.Random = *dec.Random
-	if dec.FeeRecipient == nil {
+	if dec.SuggestedFeeRecipient == nil {
-		return errors.New("missing required field 'feeRecipient' for PayloadAttributesV1")
+		return errors.New("missing required field 'suggestedFeeRecipient' for PayloadAttributesV1")
 	}
-	p.FeeRecipient = *dec.FeeRecipient
+	p.SuggestedFeeRecipient = *dec.SuggestedFeeRecipient
 	return nil
 }

core/beacon/gen_ed.go

@@ -1,6 +1,6 @@
 // Code generated by github.com/fjl/gencodec. DO NOT EDIT.
 
-package catalyst
+package beacon
 
 import (
 	"encoding/json"
@@ -17,9 +17,9 @@ var _ = (*executableDataMarshaling)(nil)
 func (e ExecutableDataV1) MarshalJSON() ([]byte, error) {
 	type ExecutableDataV1 struct {
 		ParentHash    common.Hash     `json:"parentHash"    gencodec:"required"`
-		Coinbase      common.Address  `json:"coinbase"      gencodec:"required"`
+		FeeRecipient  common.Address  `json:"feeRecipient"  gencodec:"required"`
 		StateRoot     common.Hash     `json:"stateRoot"     gencodec:"required"`
-		ReceiptRoot   common.Hash     `json:"receiptRoot"   gencodec:"required"`
+		ReceiptsRoot  common.Hash     `json:"receiptsRoot"   gencodec:"required"`
 		LogsBloom     hexutil.Bytes   `json:"logsBloom"     gencodec:"required"`
 		Random        common.Hash     `json:"random"        gencodec:"required"`
 		Number        hexutil.Uint64  `json:"blockNumber"   gencodec:"required"`
@@ -33,9 +33,9 @@ func (e ExecutableDataV1) MarshalJSON() ([]byte, error) {
 	}
 	var enc ExecutableDataV1
 	enc.ParentHash = e.ParentHash
-	enc.Coinbase = e.Coinbase
+	enc.FeeRecipient = e.FeeRecipient
 	enc.StateRoot = e.StateRoot
-	enc.ReceiptRoot = e.ReceiptRoot
+	enc.ReceiptsRoot = e.ReceiptsRoot
 	enc.LogsBloom = e.LogsBloom
 	enc.Random = e.Random
 	enc.Number = hexutil.Uint64(e.Number)
@@ -58,9 +58,9 @@ func (e ExecutableDataV1) MarshalJSON() ([]byte, error) {
 func (e *ExecutableDataV1) UnmarshalJSON(input []byte) error {
 	type ExecutableDataV1 struct {
 		ParentHash    *common.Hash    `json:"parentHash"    gencodec:"required"`
-		Coinbase      *common.Address `json:"coinbase"      gencodec:"required"`
+		FeeRecipient  *common.Address `json:"feeRecipient"  gencodec:"required"`
 		StateRoot     *common.Hash    `json:"stateRoot"     gencodec:"required"`
-		ReceiptRoot   *common.Hash    `json:"receiptRoot"   gencodec:"required"`
+		ReceiptsRoot  *common.Hash    `json:"receiptsRoot"   gencodec:"required"`
 		LogsBloom     *hexutil.Bytes  `json:"logsBloom"     gencodec:"required"`
 		Random        *common.Hash    `json:"random"        gencodec:"required"`
 		Number        *hexutil.Uint64 `json:"blockNumber"   gencodec:"required"`
@@ -80,18 +80,18 @@ func (e *ExecutableDataV1) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'parentHash' for ExecutableDataV1")
 	}
 	e.ParentHash = *dec.ParentHash
-	if dec.Coinbase == nil {
+	if dec.FeeRecipient == nil {
-		return errors.New("missing required field 'coinbase' for ExecutableDataV1")
+		return errors.New("missing required field 'feeRecipient' for ExecutableDataV1")
 	}
-	e.Coinbase = *dec.Coinbase
+	e.FeeRecipient = *dec.FeeRecipient
 	if dec.StateRoot == nil {
 		return errors.New("missing required field 'stateRoot' for ExecutableDataV1")
 	}
 	e.StateRoot = *dec.StateRoot
-	if dec.ReceiptRoot == nil {
+	if dec.ReceiptsRoot == nil {
-		return errors.New("missing required field 'receiptRoot' for ExecutableDataV1")
+		return errors.New("missing required field 'receiptsRoot' for ExecutableDataV1")
 	}
-	e.ReceiptRoot = *dec.ReceiptRoot
+	e.ReceiptsRoot = *dec.ReceiptsRoot
 	if dec.LogsBloom == nil {
 		return errors.New("missing required field 'logsBloom' for ExecutableDataV1")
 	}

core/beacon/types.go

@@ -0,0 +1,204 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package beacon
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+//go:generate go run github.com/fjl/gencodec -type PayloadAttributesV1 -field-override payloadAttributesMarshaling -out gen_blockparams.go
+
+// PayloadAttributesV1 structure described at https://github.com/ethereum/execution-apis/pull/74
+type PayloadAttributesV1 struct {
+	Timestamp             uint64         `json:"timestamp"     gencodec:"required"`
+	Random                common.Hash    `json:"random"        gencodec:"required"`
+	SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient"  gencodec:"required"`
+}
+
+// JSON type overrides for PayloadAttributesV1.
+type payloadAttributesMarshaling struct {
+	Timestamp hexutil.Uint64
+}
+
+//go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go
+
+// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md
+type ExecutableDataV1 struct {
+	ParentHash    common.Hash    `json:"parentHash"    gencodec:"required"`
+	FeeRecipient  common.Address `json:"feeRecipient"  gencodec:"required"`
+	StateRoot     common.Hash    `json:"stateRoot"     gencodec:"required"`
+	ReceiptsRoot  common.Hash    `json:"receiptsRoot"   gencodec:"required"`
+	LogsBloom     []byte         `json:"logsBloom"     gencodec:"required"`
+	Random        common.Hash    `json:"random"        gencodec:"required"`
+	Number        uint64         `json:"blockNumber"   gencodec:"required"`
+	GasLimit      uint64         `json:"gasLimit"      gencodec:"required"`
+	GasUsed       uint64         `json:"gasUsed"       gencodec:"required"`
+	Timestamp     uint64         `json:"timestamp"     gencodec:"required"`
+	ExtraData     []byte         `json:"extraData"     gencodec:"required"`
+	BaseFeePerGas *big.Int       `json:"baseFeePerGas" gencodec:"required"`
+	BlockHash     common.Hash    `json:"blockHash"     gencodec:"required"`
+	Transactions  [][]byte       `json:"transactions"  gencodec:"required"`
+}
+
+// JSON type overrides for executableData.
+type executableDataMarshaling struct {
+	Number        hexutil.Uint64
+	GasLimit      hexutil.Uint64
+	GasUsed       hexutil.Uint64
+	Timestamp     hexutil.Uint64
+	BaseFeePerGas *hexutil.Big
+	ExtraData     hexutil.Bytes
+	LogsBloom     hexutil.Bytes
+	Transactions  []hexutil.Bytes
+}
+
+type NewBlockResponse struct {
+	Valid bool `json:"valid"`
+}
+
+type GenericResponse struct {
+	Success bool `json:"success"`
+}
+
+type GenericStringResponse struct {
+	Status string `json:"status"`
+}
+
+type ExecutePayloadResponse struct {
+	Status          string      `json:"status"`
+	LatestValidHash common.Hash `json:"latestValidHash"`
+}
+
+type ConsensusValidatedParams struct {
+	BlockHash common.Hash `json:"blockHash"`
+	Status    string      `json:"status"`
+}
+
+// PayloadID is an identifier of the payload build process
+type PayloadID [8]byte
+
+func (b PayloadID) String() string {
+	return hexutil.Encode(b[:])
+}
+
+func (b PayloadID) MarshalText() ([]byte, error) {
+	return hexutil.Bytes(b[:]).MarshalText()
+}
+
+func (b *PayloadID) UnmarshalText(input []byte) error {
+	err := hexutil.UnmarshalFixedText("PayloadID", input, b[:])
+	if err != nil {
+		return fmt.Errorf("invalid payload id %q: %w", input, err)
+	}
+	return nil
+}
+
+type ForkChoiceResponse struct {
+	Status    string     `json:"status"`
+	PayloadID *PayloadID `json:"payloadId"`
+}
+
+type ForkchoiceStateV1 struct {
+	HeadBlockHash      common.Hash `json:"headBlockHash"`
+	SafeBlockHash      common.Hash `json:"safeBlockHash"`
+	FinalizedBlockHash common.Hash `json:"finalizedBlockHash"`
+}
+
+func encodeTransactions(txs []*types.Transaction) [][]byte {
+	var enc = make([][]byte, len(txs))
+	for i, tx := range txs {
+		enc[i], _ = tx.MarshalBinary()
+	}
+	return enc
+}
+
+func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) {
+	var txs = make([]*types.Transaction, len(enc))
+	for i, encTx := range enc {
+		var tx types.Transaction
+		if err := tx.UnmarshalBinary(encTx); err != nil {
+			return nil, fmt.Errorf("invalid transaction %d: %v", i, err)
+		}
+		txs[i] = &tx
+	}
+	return txs, nil
+}
+
+// ExecutableDataToBlock constructs a block from executable data.
+// It verifies that the following fields:
+// 		len(extraData) <= 32
+// 		uncleHash = emptyUncleHash
+// 		difficulty = 0
+// and that the blockhash of the constructed block matches the parameters.
+func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) {
+	txs, err := decodeTransactions(params.Transactions)
+	if err != nil {
+		return nil, err
+	}
+	if len(params.ExtraData) > 32 {
+		return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData))
+	}
+	header := &types.Header{
+		ParentHash:  params.ParentHash,
+		UncleHash:   types.EmptyUncleHash,
+		Coinbase:    params.FeeRecipient,
+		Root:        params.StateRoot,
+		TxHash:      types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
+		ReceiptHash: params.ReceiptsRoot,
+		Bloom:       types.BytesToBloom(params.LogsBloom),
+		Difficulty:  common.Big0,
+		Number:      new(big.Int).SetUint64(params.Number),
+		GasLimit:    params.GasLimit,
+		GasUsed:     params.GasUsed,
+		Time:        params.Timestamp,
+		BaseFee:     params.BaseFeePerGas,
+		Extra:       params.ExtraData,
+		MixDigest:   params.Random,
+	}
+	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */)
+	if block.Hash() != params.BlockHash {
+		return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash())
+	}
+	return block, nil
+}
+
+// BlockToExecutableData constructs the executableDataV1 structure by filling the
+// fields from the given block. It assumes the given block is post-merge block.
+func BlockToExecutableData(block *types.Block) *ExecutableDataV1 {
+	return &ExecutableDataV1{
+		BlockHash:     block.Hash(),
+		ParentHash:    block.ParentHash(),
+		FeeRecipient:  block.Coinbase(),
+		StateRoot:     block.Root(),
+		Number:        block.NumberU64(),
+		GasLimit:      block.GasLimit(),
+		GasUsed:       block.GasUsed(),
+		BaseFeePerGas: block.BaseFee(),
+		Timestamp:     block.Time(),
+		ReceiptsRoot:  block.ReceiptHash(),
+		LogsBloom:     block.Bloom().Bytes(),
+		Transactions:  encodeTransactions(block.Transactions()),
+		Random:        block.MixDigest(),
+		ExtraData:     block.Extra(),
+	}
+}

core/blockchain.go

@@ -554,7 +554,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo
 			// Degrade the chain markers if they are explicitly reverted.
 			// In theory we should update all in-memory markers in the
 			// last step, however the direction of SetHead is from high
-			// to low, so it's safe the update in-memory markers directly.
+			// to low, so it's safe to update in-memory markers directly.
 			bc.currentBlock.Store(newHeadBlock)
 			headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
 		}
@@ -979,32 +979,31 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		// range. In this case, all tx indices of newly imported blocks should be
 		// generated.
 		var batch = bc.db.NewBatch()
-		for _, block := range blockChain {
+		for i, block := range blockChain {
 			if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit {
 				rawdb.WriteTxLookupEntriesByBlock(batch, block)
 			} else if rawdb.ReadTxIndexTail(bc.db) != nil {
 				rawdb.WriteTxLookupEntriesByBlock(batch, block)
 			}
 			stats.processed++
-		}
 
-		// Flush all tx-lookup index data.
+			if batch.ValueSize() > ethdb.IdealBatchSize || i == len(blockChain)-1 {
-		size += int64(batch.ValueSize())
+				size += int64(batch.ValueSize())
-		if err := batch.Write(); err != nil {
+				if err = batch.Write(); err != nil {
-			// The tx index data could not be written.
+					fastBlock := bc.CurrentFastBlock().NumberU64()
-			// Roll back the ancient store update.
+					if err := bc.db.TruncateAncients(fastBlock + 1); err != nil {
-			fastBlock := bc.CurrentFastBlock().NumberU64()
+						log.Error("Can't truncate ancient store after failed insert", "err", err)
-			if err := bc.db.TruncateAncients(fastBlock + 1); err != nil {
+					}
-				log.Error("Can't truncate ancient store after failed insert", "err", err)
+					return 0, err
+				}
+				batch.Reset()
 			}
-			return 0, err
 		}
 
 		// Sync the ancient store explicitly to ensure all data has been flushed to disk.
 		if err := bc.db.Sync(); err != nil {
 			return 0, err
 		}
-
 		// Update the current fast block because all block data is now present in DB.
 		previousFastBlock := bc.CurrentFastBlock().NumberU64()
 		if !updateHead(blockChain[len(blockChain)-1]) {

core/blockchain_repair_test.go

@@ -1779,6 +1779,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 			SnapshotLimit:  0, // Disable snapshot by default
 		}
 	)
+	defer engine.Close()
 	if snapshots {
 		config.SnapshotLimit = 256
 		config.SnapshotWait = true
@@ -1836,25 +1837,25 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 	defer db.Close()
 
-	chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
+	newChain, err := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
-	defer chain.Stop()
+	defer newChain.Stop()
 
 	// Iterate over all the remaining blocks and ensure there are no gaps
-	verifyNoGaps(t, chain, true, canonblocks)
+	verifyNoGaps(t, newChain, true, canonblocks)
-	verifyNoGaps(t, chain, false, sideblocks)
+	verifyNoGaps(t, newChain, false, sideblocks)
-	verifyCutoff(t, chain, true, canonblocks, tt.expCanonicalBlocks)
+	verifyCutoff(t, newChain, true, canonblocks, tt.expCanonicalBlocks)
-	verifyCutoff(t, chain, false, sideblocks, tt.expSidechainBlocks)
+	verifyCutoff(t, newChain, false, sideblocks, tt.expSidechainBlocks)
 
-	if head := chain.CurrentHeader(); head.Number.Uint64() != tt.expHeadHeader {
+	if head := newChain.CurrentHeader(); head.Number.Uint64() != tt.expHeadHeader {
 		t.Errorf("Head header mismatch: have %d, want %d", head.Number, tt.expHeadHeader)
 	}
-	if head := chain.CurrentFastBlock(); head.NumberU64() != tt.expHeadFastBlock {
+	if head := newChain.CurrentFastBlock(); head.NumberU64() != tt.expHeadFastBlock {
 		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadFastBlock)
 	}
-	if head := chain.CurrentBlock(); head.NumberU64() != tt.expHeadBlock {
+	if head := newChain.CurrentBlock(); head.NumberU64() != tt.expHeadBlock {
 		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), tt.expHeadBlock)
 	}
 	if frozen, err := db.(freezer).Ancients(); err != nil {

core/blockchain_test.go

@@ -2987,10 +2987,10 @@ func TestDeleteRecreateSlots(t *testing.T) {
 	initCode := []byte{
 		byte(vm.PUSH1), 0x3, // value
 		byte(vm.PUSH1), 0x3, // location
-		byte(vm.SSTORE),     // Set slot[3] = 1
+		byte(vm.SSTORE),     // Set slot[3] = 3
 		byte(vm.PUSH1), 0x4, // value
 		byte(vm.PUSH1), 0x4, // location
-		byte(vm.SSTORE), // Set slot[4] = 1
+		byte(vm.SSTORE), // Set slot[4] = 4
 		// Slots are set, now return the code
 		byte(vm.PUSH2), byte(vm.PC), byte(vm.SELFDESTRUCT), // Push code on stack
 		byte(vm.PUSH1), 0x0, // memory start on stack

core/evm.go

@@ -40,6 +40,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common
 	var (
 		beneficiary common.Address
 		baseFee     *big.Int
+		random      *common.Hash
 	)
 
 	// If we don't have an explicit author (i.e. not mining), extract from the header
@@ -51,6 +52,9 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common
 	if header.BaseFee != nil {
 		baseFee = new(big.Int).Set(header.BaseFee)
 	}
+	if header.Difficulty.Cmp(common.Big0) == 0 {
+		random = &header.MixDigest
+	}
 	return vm.BlockContext{
 		CanTransfer: CanTransfer,
 		Transfer:    Transfer,
@@ -61,6 +65,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common
 		Difficulty:  new(big.Int).Set(header.Difficulty),
 		BaseFee:     baseFee,
 		GasLimit:    header.GasLimit,
+		Random:      random,
 	}
 }
 

core/forkid/forkid_test.go

@@ -19,6 +19,7 @@ package forkid
 import (
 	"bytes"
 	"math"
+	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -29,6 +30,8 @@ import (
 // TestCreation tests that different genesis and fork rule combinations result in
 // the correct fork ID.
 func TestCreation(t *testing.T) {
+	mergeConfig := *params.MainnetChainConfig
+	mergeConfig.MergeForkBlock = big.NewInt(15000000)
 	type testcase struct {
 		head uint64
 		want ID
@@ -65,7 +68,7 @@ func TestCreation(t *testing.T) {
 				{12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block
 				{12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block
 				{13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block
-				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}},        /// First Arrow Glacier block
+				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}},        // First Arrow Glacier block
 				{20000000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}},        // Future Arrow Glacier block
 			},
 		},
@@ -133,6 +136,38 @@ func TestCreation(t *testing.T) {
 				{6000000, ID{Hash: checksumToBytes(0xB8C6299D), Next: 0}},       // Future London block
 			},
 		},
+		// Merge test cases
+		{
+			&mergeConfig,
+			params.MainnetGenesisHash,
+			[]testcase{
+				{0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}},         // Unsynced
+				{1149999, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}},   // Last Frontier block
+				{1150000, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}},   // First Homestead block
+				{1919999, ID{Hash: checksumToBytes(0x97c2c34c), Next: 1920000}},   // Last Homestead block
+				{1920000, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}},   // First DAO block
+				{2462999, ID{Hash: checksumToBytes(0x91d1f948), Next: 2463000}},   // Last DAO block
+				{2463000, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}},   // First Tangerine block
+				{2674999, ID{Hash: checksumToBytes(0x7a64da13), Next: 2675000}},   // Last Tangerine block
+				{2675000, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}},   // First Spurious block
+				{4369999, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}},   // Last Spurious block
+				{4370000, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}},   // First Byzantium block
+				{7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}},   // Last Byzantium block
+				{7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}},   // First and last Constantinople, first Petersburg block
+				{9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}},   // Last Petersburg block
+				{9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}},   // First Istanbul and first Muir Glacier block
+				{9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}},   // Last Istanbul and first Muir Glacier block
+				{9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}},  // First Muir Glacier block
+				{12243999, ID{Hash: checksumToBytes(0xe029e991), Next: 12244000}}, // Last Muir Glacier block
+				{12244000, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // First Berlin block
+				{12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block
+				{12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block
+				{13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block
+				{13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15000000}}, // First Arrow Glacier block
+				{15000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}},        // First Merge Start block
+				{20000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}},        // Future Merge Start block
+			},
+		},
 	}
 	for i, tt := range tests {
 		for j, ttt := range tt.cases {

core/genesis.go

@@ -294,7 +294,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
 	if g.GasLimit == 0 {
 		head.GasLimit = params.GenesisGasLimit
 	}
-	if g.Difficulty == nil {
+	if g.Difficulty == nil && g.Mixhash == (common.Hash{}) {
 		head.Difficulty = params.GenesisDifficulty
 	}
 	if g.Config != nil && g.Config.IsLondon(common.Big0) {

core/rawdb/accessors_chain.go

@@ -447,8 +447,11 @@ func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
 		if len(data) > 0 {
 			return nil
 		}
-		// Get it by hash from leveldb
+		// Block is not in ancients, read from leveldb by hash and number.
-		data, _ = db.Get(blockBodyKey(number, ReadCanonicalHash(db, number)))
+		// Note: ReadCanonicalHash cannot be used here because it also
+		// calls ReadAncients internally.
+		hash, _ := db.Get(headerHashKey(number))
+		data, _ = db.Get(blockBodyKey(number, common.BytesToHash(hash)))
 		return nil
 	})
 	return data

core/rawdb/accessors_metadata.go

@@ -139,6 +139,28 @@ func PopUncleanShutdownMarker(db ethdb.KeyValueStore) {
 	}
 }
 
+// UpdateUncleanShutdownMarker updates the last marker's timestamp to now.
+func UpdateUncleanShutdownMarker(db ethdb.KeyValueStore) {
+	var uncleanShutdowns crashList
+	// Read old data
+	if data, err := db.Get(uncleanShutdownKey); err != nil {
+		log.Warn("Error reading unclean shutdown markers", "error", err)
+	} else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
+		log.Warn("Error decoding unclean shutdown markers", "error", err)
+	}
+	// This shouldn't happen because we push a marker on Backend instantiation
+	count := len(uncleanShutdowns.Recent)
+	if count == 0 {
+		log.Warn("No unclean shutdown marker to update")
+		return
+	}
+	uncleanShutdowns.Recent[count-1] = uint64(time.Now().Unix())
+	data, _ := rlp.EncodeToBytes(uncleanShutdowns)
+	if err := db.Put(uncleanShutdownKey, data); err != nil {
+		log.Warn("Failed to write unclean-shutdown marker", "err", err)
+	}
+}
+
 // ReadTransitionStatus retrieves the eth2 transition status from the database
 func ReadTransitionStatus(db ethdb.KeyValueReader) []byte {
 	data, _ := db.Get(transitionStatusKey)

core/rawdb/accessors_state.go

@@ -41,16 +41,14 @@ func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
 
 // ReadCode retrieves the contract code of the provided code hash.
 func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
-	// Try with the legacy code scheme first, if not then try with current
+	// Try with the prefixed code scheme first, if not then try with legacy
-	// scheme. Since most of the code will be found with legacy scheme.
+	// scheme.
-	//
+	data := ReadCodeWithPrefix(db, hash)
-	// todo(rjl493456442) change the order when we forcibly upgrade the code
-	// scheme with snapshot.
-	data, _ := db.Get(hash[:])
 	if len(data) != 0 {
 		return data
 	}
-	return ReadCodeWithPrefix(db, hash)
+	data, _ = db.Get(hash[:])
+	return data
 }
 
 // ReadCodeWithPrefix retrieves the contract code of the provided code hash.
@@ -61,6 +59,14 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
+// HasCodeWithPrefix checks if the contract code corresponding to the
+// provided code hash is present in the db. This function will only check
+// presence using the prefix-scheme.
+func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
+	ok, _ := db.Has(codeKey(hash))
+	return ok
+}
+
 // WriteCode writes the provided contract code database.
 func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
 	if err := db.Put(codeKey(hash), code); err != nil {
@@ -81,6 +87,12 @@ func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
+// HasTrieNode checks if the trie node with the provided hash is present in db.
+func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
+	ok, _ := db.Has(hash.Bytes())
+	return ok
+}
+
 // WriteTrieNode writes the provided trie node database.
 func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
 	if err := db.Put(hash.Bytes(), node); err != nil {

core/rawdb/freezer.go

@@ -133,7 +133,7 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
 
 	// Create the tables.
 	for name, disableSnappy := range tables {
-		table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy)
+		table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
 		if err != nil {
 			for _, table := range freezer.tables {
 				table.Close()
@@ -144,8 +144,15 @@ func newFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
 		freezer.tables[name] = table
 	}
 
-	// Truncate all tables to common length.
+	if freezer.readonly {
-	if err := freezer.repair(); err != nil {
+		// In readonly mode only validate, don't truncate.
+		// validate also sets `freezer.frozen`.
+		err = freezer.validate()
+	} else {
+		// Truncate all tables to common length.
+		err = freezer.repair()
+	}
+	if err != nil {
 		for _, table := range freezer.tables {
 			table.Close()
 		}
@@ -308,6 +315,33 @@ func (f *freezer) Sync() error {
 	return nil
 }
 
+// validate checks that every table has the same length.
+// Used instead of `repair` in readonly mode.
+func (f *freezer) validate() error {
+	if len(f.tables) == 0 {
+		return nil
+	}
+	var (
+		length uint64
+		name   string
+	)
+	// Hack to get length of any table
+	for kind, table := range f.tables {
+		length = atomic.LoadUint64(&table.items)
+		name = kind
+		break
+	}
+	// Now check every table against that length
+	for kind, table := range f.tables {
+		items := atomic.LoadUint64(&table.items)
+		if length != items {
+			return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length)
+		}
+	}
+	atomic.StoreUint64(&f.frozen, length)
+	return nil
+}
+
 // repair truncates all data tables to the same length.
 func (f *freezer) repair() error {
 	min := uint64(math.MaxUint64)

core/rawdb/freezer_table.go

@@ -94,7 +94,8 @@ type freezerTable struct {
 	// so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG).
 	items uint64 // Number of items stored in the table (including items removed from tail)
 
-	noCompression bool   // if true, disables snappy compression. Note: does not work retroactively
+	noCompression bool // if true, disables snappy compression. Note: does not work retroactively
+	readonly      bool
 	maxFileSize   uint32 // Max file size for data-files
 	name          string
 	path          string
@@ -119,8 +120,8 @@ type freezerTable struct {
 }
 
 // NewFreezerTable opens the given path as a freezer table.
-func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) {
+func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
-	return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy)
+	return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
 }
 
 // openFreezerFileForAppend opens a freezer table file and seeks to the end
@@ -164,7 +165,7 @@ func truncateFreezerFile(file *os.File, size int64) error {
 // newTable opens a freezer table, creating the data and index files if they are
 // non existent. Both files are truncated to the shortest common length to ensure
 // they don't go out of sync.
-func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
+func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) {
 	// Ensure the containing directory exists and open the indexEntry file
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return nil, err
@@ -177,7 +178,16 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
 		// Compressed idx
 		idxName = fmt.Sprintf("%s.cidx", name)
 	}
-	offsets, err := openFreezerFileForAppend(filepath.Join(path, idxName))
+	var (
+		err     error
+		offsets *os.File
+	)
+	if readonly {
+		// Will fail if table doesn't exist
+		offsets, err = openFreezerFileForReadOnly(filepath.Join(path, idxName))
+	} else {
+		offsets, err = openFreezerFileForAppend(filepath.Join(path, idxName))
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -192,6 +202,7 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
 		path:          path,
 		logger:        log.New("database", path, "table", name),
 		noCompression: noCompression,
+		readonly:      readonly,
 		maxFileSize:   maxFilesize,
 	}
 	if err := tab.repair(); err != nil {
@@ -252,7 +263,11 @@ func (t *freezerTable) repair() error {
 
 	t.index.ReadAt(buffer, offsetsSize-indexEntrySize)
 	lastIndex.unmarshalBinary(buffer)
-	t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForAppend)
+	if t.readonly {
+		t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly)
+	} else {
+		t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForAppend)
+	}
 	if err != nil {
 		return err
 	}
@@ -301,12 +316,15 @@ func (t *freezerTable) repair() error {
 			contentExp = int64(lastIndex.offset)
 		}
 	}
-	// Ensure all reparation changes have been written to disk
+	// Sync() fails for read-only files on windows.
-	if err := t.index.Sync(); err != nil {
+	if !t.readonly {
-		return err
+		// Ensure all reparation changes have been written to disk
-	}
+		if err := t.index.Sync(); err != nil {
-	if err := t.head.Sync(); err != nil {
+			return err
-		return err
+		}
+		if err := t.head.Sync(); err != nil {
+			return err
+		}
 	}
 	// Update the item and byte counters and return
 	t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file
@@ -334,8 +352,12 @@ func (t *freezerTable) preopen() (err error) {
 			return err
 		}
 	}
-	// Open head in read/write
+	if t.readonly {
-	t.head, err = t.openFile(t.headId, openFreezerFileForAppend)
+		t.head, err = t.openFile(t.headId, openFreezerFileForReadOnly)
+	} else {
+		// Open head in read/write
+		t.head, err = t.openFile(t.headId, openFreezerFileForAppend)
+	}
 	return err
 }
 

core/rawdb/freezer_table_test.go

@@ -40,7 +40,7 @@ func TestFreezerBasics(t *testing.T) {
 	// set cutoff at 50 bytes
 	f, err := newTable(os.TempDir(),
 		fmt.Sprintf("unittest-%d", rand.Uint64()),
-		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true)
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -85,7 +85,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
 		f          *freezerTable
 		err        error
 	)
-	f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+	f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -99,7 +99,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
 		require.NoError(t, batch.commit())
 		f.Close()
 
-		f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -116,7 +116,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
 			t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
 		}
 		f.Close()
-		f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -131,7 +131,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
 
 	// Fill table
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -160,7 +160,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
 
 	// Now open it again
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -183,7 +183,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 
 	// Fill a table and close it
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -209,7 +209,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 
 	// Now open it again
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -232,7 +232,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
 
 	// And if we open it, we should now be able to read all of them (new values)
 	{
-		f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		for y := 1; y < 255; y++ {
 			exp := getChunk(15, ^y)
 			got, err := f.Retrieve(uint64(y))
@@ -254,7 +254,7 @@ func TestSnappyDetection(t *testing.T) {
 
 	// Open with snappy
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -265,7 +265,7 @@ func TestSnappyDetection(t *testing.T) {
 
 	// Open without snappy
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
 
 	// Open with snappy
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -309,7 +309,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
 
 	// Fill a table and close it
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -345,7 +345,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
 	// 45, 45, 15
 	// with 3+3+1 items
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -366,7 +366,7 @@ func TestFreezerTruncate(t *testing.T) {
 
 	// Fill table
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -382,7 +382,7 @@ func TestFreezerTruncate(t *testing.T) {
 
 	// Reopen, truncate
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -407,7 +407,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
 
 	// Fill table
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -440,7 +440,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
 
 	// Reopen
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -475,7 +475,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
 
 	// Fill table
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -491,7 +491,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
 
 	// Reopen and read all files
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -523,7 +523,7 @@ func TestFreezerOffset(t *testing.T) {
 
 	// Fill table
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -584,7 +584,7 @@ func TestFreezerOffset(t *testing.T) {
 
 	// Now open again
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -638,7 +638,7 @@ func TestFreezerOffset(t *testing.T) {
 
 	// Check that existing items have been moved to index 1M.
 	{
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -726,7 +726,7 @@ func TestSequentialRead(t *testing.T) {
 	rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
 	fname := fmt.Sprintf("batchread-%d", rand.Uint64())
 	{ // Fill table
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -736,7 +736,7 @@ func TestSequentialRead(t *testing.T) {
 		f.Close()
 	}
 	{ // Open it, iterate, verify iteration
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -757,7 +757,7 @@ func TestSequentialRead(t *testing.T) {
 	}
 	{ // Open it, iterate, verify byte limit. The byte limit is less than item
 		// size, so each lookup should only return one item
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -786,7 +786,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
 	rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
 	fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
 	{ // Fill table
-		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
+		f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -808,7 +808,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
 		{100, 109, 10},
 	} {
 		{
-			f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
+			f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -829,3 +829,89 @@ func TestSequentialReadByteLimit(t *testing.T) {
 		}
 	}
 }
+
+func TestFreezerReadonly(t *testing.T) {
+	tmpdir := os.TempDir()
+	// Case 1: Check it fails on non-existent file.
+	_, err := newTable(tmpdir,
+		fmt.Sprintf("readonlytest-%d", rand.Uint64()),
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
+	if err == nil {
+		t.Fatal("readonly table instantiation should fail for non-existent table")
+	}
+
+	// Case 2: Check that it fails on invalid index length.
+	fname := fmt.Sprintf("readonlytest-%d", rand.Uint64())
+	idxFile, err := openFreezerFileForAppend(filepath.Join(tmpdir, fmt.Sprintf("%s.ridx", fname)))
+	if err != nil {
+		t.Errorf("Failed to open index file: %v\n", err)
+	}
+	// size should not be a multiple of indexEntrySize.
+	idxFile.Write(make([]byte, 17))
+	idxFile.Close()
+	_, err = newTable(tmpdir, fname,
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
+	if err == nil {
+		t.Errorf("readonly table instantiation should fail for invalid index size")
+	}
+
+	// Case 3: Open table non-readonly table to write some data.
+	// Then corrupt the head file and make sure opening the table
+	// again in readonly triggers an error.
+	fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
+	f, err := newTable(tmpdir, fname,
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
+	if err != nil {
+		t.Fatalf("failed to instantiate table: %v", err)
+	}
+	writeChunks(t, f, 8, 32)
+	// Corrupt table file
+	if _, err := f.head.Write([]byte{1, 1}); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	_, err = newTable(tmpdir, fname,
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
+	if err == nil {
+		t.Errorf("readonly table instantiation should fail for corrupt table file")
+	}
+
+	// Case 4: Write some data to a table and later re-open it as readonly.
+	// Should be successful.
+	fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
+	f, err = newTable(tmpdir, fname,
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
+	if err != nil {
+		t.Fatalf("failed to instantiate table: %v\n", err)
+	}
+	writeChunks(t, f, 32, 128)
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	f, err = newTable(tmpdir, fname,
+		metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	v, err := f.Retrieve(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	exp := getChunk(128, 10)
+	if !bytes.Equal(v, exp) {
+		t.Errorf("retrieved value is incorrect")
+	}
+
+	// Case 5: Now write some data via a batch.
+	// This should fail either during AppendRaw or Commit
+	batch := f.newBatch()
+	writeErr := batch.AppendRaw(32, make([]byte, 1))
+	if writeErr == nil {
+		writeErr = batch.commit()
+	}
+	if writeErr == nil {
+		t.Fatalf("Writing to readonly table should fail")
+	}
+}

core/rawdb/freezer_test.go

@@ -253,6 +253,44 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
 	}
 }
 
+func TestFreezerReadonlyValidate(t *testing.T) {
+	tables := map[string]bool{"a": true, "b": true}
+	dir, err := ioutil.TempDir("", "freezer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	// Open non-readonly freezer and fill individual tables
+	// with different amount of data.
+	f, err := newFreezer(dir, "", false, 2049, tables)
+	if err != nil {
+		t.Fatal("can't open freezer", err)
+	}
+	var item = make([]byte, 1024)
+	aBatch := f.tables["a"].newBatch()
+	require.NoError(t, aBatch.AppendRaw(0, item))
+	require.NoError(t, aBatch.AppendRaw(1, item))
+	require.NoError(t, aBatch.AppendRaw(2, item))
+	require.NoError(t, aBatch.commit())
+	bBatch := f.tables["b"].newBatch()
+	require.NoError(t, bBatch.AppendRaw(0, item))
+	require.NoError(t, bBatch.commit())
+	if f.tables["a"].items != 3 {
+		t.Fatalf("unexpected number of items in table")
+	}
+	if f.tables["b"].items != 1 {
+		t.Fatalf("unexpected number of items in table")
+	}
+	require.NoError(t, f.Close())
+
+	// Re-openening as readonly should fail when validating
+	// table lengths.
+	f, err = newFreezer(dir, "", true, 2049, tables)
+	if err == nil {
+		t.Fatal("readonly freezer should fail with differing table lengths")
+	}
+}
+
 func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) {
 	t.Helper()
 

core/state/snapshot/journal.go

@@ -66,6 +66,29 @@ type journalStorage struct {
 	Vals [][]byte
 }
 
+func ParseGeneratorStatus(generatorBlob []byte) string {
+	if len(generatorBlob) == 0 {
+		return ""
+	}
+	var generator journalGenerator
+	if err := rlp.DecodeBytes(generatorBlob, &generator); err != nil {
+		log.Warn("failed to decode snapshot generator", "err", err)
+		return ""
+	}
+	// Figure out whether we're after or within an account
+	var m string
+	switch marker := generator.Marker; len(marker) {
+	case common.HashLength:
+		m = fmt.Sprintf("at %#x", marker)
+	case 2 * common.HashLength:
+		m = fmt.Sprintf("in %#x at %#x", marker[:common.HashLength], marker[common.HashLength:])
+	default:
+		m = fmt.Sprintf("%#x", marker)
+	}
+	return fmt.Sprintf(`Done: %v, Accounts: %d, Slots: %d, Storage: %d, Marker: %s`,
+		generator.Done, generator.Accounts, generator.Slots, generator.Storage, m)
+}
+
 // loadAndParseJournal tries to parse the snapshot journal in latest format.
 func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, journalGenerator, error) {
 	// Retrieve the disk layer generator. It must exist, no matter the

core/state_transition.go

@@ -310,7 +310,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
 	}
 
 	// Set up the initial access list.
-	if rules := st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber); rules.IsBerlin {
+	if rules := st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber, st.evm.Context.Random != nil); rules.IsBerlin {
 		st.state.PrepareAccessList(msg.From(), msg.To(), vm.ActivePrecompiles(rules), msg.AccessList())
 	}
 	var (

core/tx_pool.go

@@ -621,9 +621,8 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error {
 	if err != nil {
 		return ErrInvalidSender
 	}
-	// Drop non-local transactions under our own minimal accepted gas price or tip.
+	// Drop non-local transactions under our own minimal accepted gas price or tip
-	pendingBaseFee := pool.priced.urgent.baseFee
+	if !local && tx.GasTipCapIntCmp(pool.gasPrice) < 0 {
-	if !local && tx.EffectiveGasTipIntCmp(pool.gasPrice, pendingBaseFee) < 0 {
 		return ErrUnderpriced
 	}
 	// Ensure the transaction adheres to nonce ordering

core/types/dynamic_fee_tx.go

@@ -25,8 +25,8 @@ import (
 type DynamicFeeTx struct {
 	ChainID    *big.Int
 	Nonce      uint64
-	GasTipCap  *big.Int
+	GasTipCap  *big.Int // a.k.a. maxPriorityFeePerGas
-	GasFeeCap  *big.Int
+	GasFeeCap  *big.Int // a.k.a. maxFeePerGas
 	Gas        uint64
 	To         *common.Address `rlp:"nil"` // nil means contract creation
 	Value      *big.Int

core/vm/analysis.go

@@ -17,12 +17,12 @@
 package vm
 
 const (
-	set2BitsMask = uint16(0b1100_0000_0000_0000)
+	set2BitsMask = uint16(0b11)
-	set3BitsMask = uint16(0b1110_0000_0000_0000)
+	set3BitsMask = uint16(0b111)
-	set4BitsMask = uint16(0b1111_0000_0000_0000)
+	set4BitsMask = uint16(0b1111)
-	set5BitsMask = uint16(0b1111_1000_0000_0000)
+	set5BitsMask = uint16(0b1_1111)
-	set6BitsMask = uint16(0b1111_1100_0000_0000)
+	set6BitsMask = uint16(0b11_1111)
-	set7BitsMask = uint16(0b1111_1110_0000_0000)
+	set7BitsMask = uint16(0b111_1111)
 )
 
 // bitvec is a bit vector which maps bytes in a program.
@@ -30,32 +30,26 @@ const (
 // it's data (i.e. argument of PUSHxx).
 type bitvec []byte
 
-var lookup = [8]byte{
-	0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1,
-}
-
 func (bits bitvec) set1(pos uint64) {
-	bits[pos/8] |= lookup[pos%8]
+	bits[pos/8] |= 1 << (pos % 8)
 }
 
 func (bits bitvec) setN(flag uint16, pos uint64) {
-	a := flag >> (pos % 8)
+	a := flag << (pos % 8)
-	bits[pos/8] |= byte(a >> 8)
+	bits[pos/8] |= byte(a)
-	if b := byte(a); b != 0 {
+	if b := byte(a >> 8); b != 0 {
-		//	If the bit-setting affects the neighbouring byte, we can assign - no need to OR it,
-		//	since it's the first write to that byte
 		bits[pos/8+1] = b
 	}
 }
 
 func (bits bitvec) set8(pos uint64) {
-	a := byte(0xFF >> (pos % 8))
+	a := byte(0xFF << (pos % 8))
 	bits[pos/8] |= a
 	bits[pos/8+1] = ^a
 }
 
 func (bits bitvec) set16(pos uint64) {
-	a := byte(0xFF >> (pos % 8))
+	a := byte(0xFF << (pos % 8))
 	bits[pos/8] |= a
 	bits[pos/8+1] = 0xFF
 	bits[pos/8+2] = ^a
@@ -63,7 +57,7 @@ func (bits bitvec) set16(pos uint64) {
 
 // codeSegment checks if the position is in a code segment.
 func (bits *bitvec) codeSegment(pos uint64) bool {
-	return ((*bits)[pos/8] & (0x80 >> (pos % 8))) == 0
+	return (((*bits)[pos/8] >> (pos % 8)) & 1) == 0
 }
 
 // codeBitmap collects data locations in code.

core/vm/analysis_test.go

@@ -17,6 +17,7 @@
 package vm
 
 import (
+	"math/bits"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/crypto"
@@ -28,24 +29,27 @@ func TestJumpDestAnalysis(t *testing.T) {
 		exp   byte
 		which int
 	}{
-		{[]byte{byte(PUSH1), 0x01, 0x01, 0x01}, 0x40, 0},
+		{[]byte{byte(PUSH1), 0x01, 0x01, 0x01}, 0b0000_0010, 0},
-		{[]byte{byte(PUSH1), byte(PUSH1), byte(PUSH1), byte(PUSH1)}, 0x50, 0},
+		{[]byte{byte(PUSH1), byte(PUSH1), byte(PUSH1), byte(PUSH1)}, 0b0000_1010, 0},
-		{[]byte{byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), 0x01, 0x01, 0x01}, 0x7F, 0},
+		{[]byte{0x00, byte(PUSH1), 0x00, byte(PUSH1), 0x00, byte(PUSH1), 0x00, byte(PUSH1)}, 0b0101_0100, 0},
-		{[]byte{byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x80, 1},
+		{[]byte{byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), byte(PUSH8), 0x01, 0x01, 0x01}, bits.Reverse8(0x7F), 0},
-		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), byte(PUSH2), byte(PUSH2), 0x01, 0x01, 0x01}, 0x03, 0},
+		{[]byte{byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0000_0001, 1},
-		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), 0x01, 0x01, 0x01, 0x01, 0x01}, 0x00, 1},
+		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), byte(PUSH2), byte(PUSH2), 0x01, 0x01, 0x01}, 0b1100_0000, 0},
-		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x74, 0},
+		{[]byte{0x01, 0x01, 0x01, 0x01, 0x01, byte(PUSH2), 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0000_0000, 1},
-		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x00, 1},
+		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0010_1110, 0},
-		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x3F, 0},
+		{[]byte{byte(PUSH3), 0x01, 0x01, 0x01, byte(PUSH1), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0000_0000, 1},
-		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0xC0, 1},
+		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b1111_1100, 0},
-		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x7F, 0},
+		{[]byte{0x01, byte(PUSH8), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0000_0011, 1},
-		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0xFF, 1},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b1111_1110, 0},
-		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0x80, 2},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b1111_1111, 1},
-		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0x7f, 0},
+		{[]byte{byte(PUSH16), 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, 0b0000_0001, 2},
-		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0xA0, 1},
+		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0b1111_1110, 0},
-		{[]byte{byte(PUSH32)}, 0x7F, 0},
+		{[]byte{byte(PUSH8), 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, byte(PUSH1), 0x01}, 0b0000_0101, 1},
-		{[]byte{byte(PUSH32)}, 0xFF, 1},
+		{[]byte{byte(PUSH32)}, 0b1111_1110, 0},
-		{[]byte{byte(PUSH32)}, 0xFF, 2},
+		{[]byte{byte(PUSH32)}, 0b1111_1111, 1},
+		{[]byte{byte(PUSH32)}, 0b1111_1111, 2},
+		{[]byte{byte(PUSH32)}, 0b1111_1111, 3},
+		{[]byte{byte(PUSH32)}, 0b0000_0001, 4},
 	}
 	for i, test := range tests {
 		ret := codeBitmap(test.code)

core/vm/evm.go

@@ -75,6 +75,7 @@ type BlockContext struct {
 	Time        *big.Int       // Provides information for TIME
 	Difficulty  *big.Int       // Provides information for DIFFICULTY
 	BaseFee     *big.Int       // Provides information for BASEFEE
+	Random      *common.Hash   // Provides information for RANDOM
 }
 
 // TxContext provides the EVM with information about a transaction.
@@ -131,7 +132,7 @@ func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig
 		StateDB:     statedb,
 		Config:      config,
 		chainConfig: chainConfig,
-		chainRules:  chainConfig.Rules(blockCtx.BlockNumber),
+		chainRules:  chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Random != nil),
 	}
 	evm.interpreter = NewEVMInterpreter(evm, config)
 	return evm

core/vm/instructions.go

@@ -477,6 +477,12 @@ func opDifficulty(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
 	return nil, nil
 }
 
+func opRandom(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
+	v := new(uint256.Int).SetBytes((interpreter.evm.Context.Random.Bytes()))
+	scope.Stack.push(v)
+	return nil, nil
+}
+
 func opGasLimit(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
 	scope.Stack.push(new(uint256.Int).SetUint64(interpreter.evm.Context.GasLimit))
 	return nil, nil

core/vm/instructions_test.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -654,3 +655,36 @@ func TestCreate2Addreses(t *testing.T) {
 		}
 	}
 }
+
+func TestRandom(t *testing.T) {
+	type testcase struct {
+		name   string
+		random common.Hash
+	}
+
+	for _, tt := range []testcase{
+		{name: "empty hash", random: common.Hash{}},
+		{name: "1", random: common.Hash{0}},
+		{name: "emptyCodeHash", random: emptyCodeHash},
+		{name: "hash(0x010203)", random: crypto.Keccak256Hash([]byte{0x01, 0x02, 0x03})},
+	} {
+		var (
+			env            = NewEVM(BlockContext{Random: &tt.random}, TxContext{}, nil, params.TestChainConfig, Config{})
+			stack          = newstack()
+			pc             = uint64(0)
+			evmInterpreter = env.interpreter
+		)
+		opRandom(&pc, evmInterpreter, &ScopeContext{nil, stack, nil})
+		if len(stack.data) != 1 {
+			t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data))
+		}
+		actual := stack.pop()
+		expected, overflow := uint256.FromBig(new(big.Int).SetBytes(tt.random.Bytes()))
+		if overflow {
+			t.Errorf("Testcase %v: invalid overflow", tt.name)
+		}
+		if actual.Cmp(expected) != 0 {
+			t.Errorf("Testcase %v: expected  %x, got %x", tt.name, expected, actual)
+		}
+	}
+}

core/vm/interpreter.go

@@ -69,6 +69,8 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter {
 	// If jump table was not initialised we set the default one.
 	if cfg.JumpTable == nil {
 		switch {
+		case evm.chainRules.IsMerge:
+			cfg.JumpTable = &mergeInstructionSet
 		case evm.chainRules.IsLondon:
 			cfg.JumpTable = &londonInstructionSet
 		case evm.chainRules.IsBerlin:

core/vm/jump_table.go

@@ -54,6 +54,7 @@ var (
 	istanbulInstructionSet         = newIstanbulInstructionSet()
 	berlinInstructionSet           = newBerlinInstructionSet()
 	londonInstructionSet           = newLondonInstructionSet()
+	mergeInstructionSet            = newMergeInstructionSet()
 )
 
 // JumpTable contains the EVM opcodes supported at a given fork.
@@ -77,6 +78,17 @@ func validate(jt JumpTable) JumpTable {
 	return jt
 }
 
+func newMergeInstructionSet() JumpTable {
+	instructionSet := newLondonInstructionSet()
+	instructionSet[RANDOM] = &operation{
+		execute:     opRandom,
+		constantGas: GasQuickStep,
+		minStack:    minStack(0, 1),
+		maxStack:    maxStack(0, 1),
+	}
+	return validate(instructionSet)
+}
+
 // newLondonInstructionSet returns the frontier, homestead, byzantium,
 // contantinople, istanbul, petersburg, berlin and london instructions.
 func newLondonInstructionSet() JumpTable {

core/vm/opcodes.go

@@ -95,6 +95,7 @@ const (
 	TIMESTAMP   OpCode = 0x42
 	NUMBER      OpCode = 0x43
 	DIFFICULTY  OpCode = 0x44
+	RANDOM      OpCode = 0x44 // Same as DIFFICULTY
 	GASLIMIT    OpCode = 0x45
 	CHAINID     OpCode = 0x46
 	SELFBALANCE OpCode = 0x47
@@ -213,6 +214,7 @@ const (
 
 	STATICCALL   OpCode = 0xfa
 	REVERT       OpCode = 0xfd
+	INVALID      OpCode = 0xfe
 	SELFDESTRUCT OpCode = 0xff
 )
 
@@ -274,7 +276,7 @@ var opCodeToString = map[OpCode]string{
 	COINBASE:    "COINBASE",
 	TIMESTAMP:   "TIMESTAMP",
 	NUMBER:      "NUMBER",
-	DIFFICULTY:  "DIFFICULTY",
+	DIFFICULTY:  "DIFFICULTY", // TODO (MariusVanDerWijden) rename to RANDOM post merge
 	GASLIMIT:    "GASLIMIT",
 	CHAINID:     "CHAINID",
 	SELFBALANCE: "SELFBALANCE",
@@ -378,6 +380,7 @@ var opCodeToString = map[OpCode]string{
 	CREATE2:      "CREATE2",
 	STATICCALL:   "STATICCALL",
 	REVERT:       "REVERT",
+	INVALID:      "INVALID",
 	SELFDESTRUCT: "SELFDESTRUCT",
 }
 
@@ -532,6 +535,7 @@ var stringToOp = map[string]OpCode{
 	"RETURN":         RETURN,
 	"CALLCODE":       CALLCODE,
 	"REVERT":         REVERT,
+	"INVALID":        INVALID,
 	"SELFDESTRUCT":   SELFDESTRUCT,
 }
 

core/vm/runtime/runtime.go

@@ -118,7 +118,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		vmenv   = NewEnv(cfg)
 		sender  = vm.AccountRef(cfg.Origin)
 	)
-	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin {
+	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil); rules.IsBerlin {
 		cfg.State.PrepareAccessList(cfg.Origin, &address, vm.ActivePrecompiles(rules), nil)
 	}
 	cfg.State.CreateAccount(address)
@@ -150,7 +150,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) {
 		vmenv  = NewEnv(cfg)
 		sender = vm.AccountRef(cfg.Origin)
 	)
-	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin {
+	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil); rules.IsBerlin {
 		cfg.State.PrepareAccessList(cfg.Origin, nil, vm.ActivePrecompiles(rules), nil)
 	}
 	// Call the code with the given configuration.
@@ -176,7 +176,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, uint64, er
 	sender := cfg.State.GetOrNewStateObject(cfg.Origin)
 	statedb := cfg.State
 
-	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber); rules.IsBerlin {
+	if rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil); rules.IsBerlin {
 		statedb.PrepareAccessList(cfg.Origin, &address, vm.ActivePrecompiles(rules), nil)
 	}
 	// Call the code with the given configuration.

crypto/ecies/ecies_test.go

@@ -279,7 +279,7 @@ var testCases = []testCase{
 	{
 		Curve:    elliptic.P384(),
 		Name:     "P384",
-		Expected: ECIES_AES256_SHA384,
+		Expected: ECIES_AES192_SHA384,
 	},
 	{
 		Curve:    elliptic.P521(),

crypto/ecies/params.go

@@ -80,6 +80,14 @@ var (
 		KeyLen:    16,
 	}
 
+	ECIES_AES192_SHA384 = &ECIESParams{
+		Hash:      sha512.New384,
+		hashAlgo:  crypto.SHA384,
+		Cipher:    aes.NewCipher,
+		BlockSize: aes.BlockSize,
+		KeyLen:    24,
+	}
+
 	ECIES_AES256_SHA256 = &ECIESParams{
 		Hash:      sha256.New,
 		hashAlgo:  crypto.SHA256,
@@ -108,7 +116,7 @@ var (
 var paramsFromCurve = map[elliptic.Curve]*ECIESParams{
 	ethcrypto.S256(): ECIES_AES128_SHA256,
 	elliptic.P256():  ECIES_AES128_SHA256,
-	elliptic.P384():  ECIES_AES256_SHA384,
+	elliptic.P384():  ECIES_AES192_SHA384,
 	elliptic.P521():  ECIES_AES256_SHA512,
 }
 

eth/backend.go

@@ -47,6 +47,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/internal/shutdowncheck"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/miner"
 	"github.com/ethereum/go-ethereum/node"
@@ -97,6 +98,8 @@ type Ethereum struct {
 	p2pServer *p2p.Server
 
 	lock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)
+
+	shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
 }
 
 // New creates a new Ethereum object (including the
@@ -157,6 +160,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		bloomRequests:     make(chan chan *bloombits.Retrieval),
 		bloomIndexer:      core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
 		p2pServer:         stack.Server(),
+		shutdownTracker:   shutdowncheck.NewShutdownTracker(chainDb),
 	}
 
 	bcVersion := rawdb.ReadDatabaseVersion(chainDb)
@@ -230,7 +234,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		return nil, err
 	}
 
-	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock, merger)
+	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock)
 	eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
 
 	eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
@@ -262,19 +266,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	stack.RegisterProtocols(eth.Protocols())
 	stack.RegisterLifecycle(eth)
 
-	// Check for unclean shutdown
+	// Successful startup; push a marker and check previous unclean shutdowns.
-	if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(chainDb); err != nil {
+	eth.shutdownTracker.MarkStartup()
-		log.Error("Could not update unclean-shutdown-marker list", "error", err)
+
-	} else {
-		if discards > 0 {
-			log.Warn("Old unclean shutdowns found", "count", discards)
-		}
-		for _, tstamp := range uncleanShutdowns {
-			t := time.Unix(int64(tstamp), 0)
-			log.Warn("Unclean shutdown detected", "booted", t,
-				"age", common.PrettyAge(t))
-		}
-	}
 	return eth, nil
 }
 
@@ -549,6 +543,9 @@ func (s *Ethereum) Start() error {
 	// Start the bloom bits servicing goroutines
 	s.startBloomHandlers(params.BloomBitsBlocks)
 
+	// Regularly update shutdown marker
+	s.shutdownTracker.Start()
+
 	// Figure out a max peers count based on the server limits
 	maxPeers := s.p2pServer.MaxPeers
 	if s.config.LightServ > 0 {
@@ -577,7 +574,10 @@ func (s *Ethereum) Stop() error {
 	s.miner.Close()
 	s.blockchain.Stop()
 	s.engine.Close()
-	rawdb.PopUncleanShutdownMarker(s.chainDb)
+
+	// Clean shutdown marker as the last thing before closing db
+	s.shutdownTracker.Stop()
+
 	s.chainDb.Close()
 	s.eventMux.Stop()
 

eth/catalyst/api.go

@@ -20,36 +20,15 @@ package catalyst
 import (
 	"crypto/sha256"
 	"encoding/binary"
-	"errors"
 	"fmt"
-	"math/big"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/beacon"
-	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/consensus/beacon"
-	"github.com/ethereum/go-ethereum/consensus/misc"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/les"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	chainParams "github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/trie"
-)
-
-var (
-	VALID            = GenericStringResponse{"VALID"}
-	SUCCESS          = GenericStringResponse{"SUCCESS"}
-	INVALID          = ForkChoiceResponse{Status: "INVALID", PayloadID: nil}
-	SYNCING          = ForkChoiceResponse{Status: "INVALID", PayloadID: nil}
-	UnknownHeader    = rpc.CustomError{Code: -32000, Message: "unknown header"}
-	UnknownPayload   = rpc.CustomError{Code: -32001, Message: "unknown payload"}
-	InvalidPayloadID = rpc.CustomError{Code: 1, Message: "invalid payload id"}
 )
 
 // Register adds catalyst APIs to the full node.
@@ -59,21 +38,7 @@ func Register(stack *node.Node, backend *eth.Ethereum) error {
 		{
 			Namespace: "engine",
 			Version:   "1.0",
-			Service:   NewConsensusAPI(backend, nil),
+			Service:   NewConsensusAPI(backend),
-			Public:    true,
-		},
-	})
-	return nil
-}
-
-// RegisterLight adds catalyst APIs to the light client.
-func RegisterLight(stack *node.Node, backend *les.LightEthereum) error {
-	log.Warn("Catalyst mode enabled", "protocol", "les")
-	stack.RegisterAPIs([]rpc.API{
-		{
-			Namespace: "engine",
-			Version:   "1.0",
-			Service:   NewConsensusAPI(nil, backend),
 			Public:    true,
 		},
 	})
@@ -81,184 +46,86 @@ func RegisterLight(stack *node.Node, backend *les.LightEthereum) error {
 }
 
 type ConsensusAPI struct {
-	light          bool
 	eth            *eth.Ethereum
-	les            *les.LightEthereum
+	preparedBlocks *payloadQueue // preparedBlocks caches payloads (*ExecutableDataV1) by payload ID (PayloadID)
-	engine         consensus.Engine // engine is the post-merge consensus engine, only for block creation
-	preparedBlocks map[uint64]*ExecutableDataV1
 }
 
-func NewConsensusAPI(eth *eth.Ethereum, les *les.LightEthereum) *ConsensusAPI {
+// NewConsensusAPI creates a new consensus api for the given backend.
-	var engine consensus.Engine
+// The underlying blockchain needs to have a valid terminal total difficulty set.
-	if eth == nil {
+func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI {
-		if les.BlockChain().Config().TerminalTotalDifficulty == nil {
+	if eth.BlockChain().Config().TerminalTotalDifficulty == nil {
-			panic("Catalyst started without valid total difficulty")
+		panic("Catalyst started without valid total difficulty")
-		}
-		if b, ok := les.Engine().(*beacon.Beacon); ok {
-			engine = beacon.New(b.InnerEngine())
-		} else {
-			engine = beacon.New(les.Engine())
-		}
-	} else {
-		if eth.BlockChain().Config().TerminalTotalDifficulty == nil {
-			panic("Catalyst started without valid total difficulty")
-		}
-		if b, ok := eth.Engine().(*beacon.Beacon); ok {
-			engine = beacon.New(b.InnerEngine())
-		} else {
-			engine = beacon.New(eth.Engine())
-		}
 	}
 	return &ConsensusAPI{
-		light:          eth == nil,
 		eth:            eth,
-		les:            les,
+		preparedBlocks: newPayloadQueue(),
-		engine:         engine,
-		preparedBlocks: make(map[uint64]*ExecutableDataV1),
 	}
 }
 
-// blockExecutionEnv gathers all the data required to execute
+// ForkchoiceUpdatedV1 has several responsibilities:
-// a block, either when assembling it or when inserting it.
+// If the method is called with an empty head block:
-type blockExecutionEnv struct {
+// 		we return success, which can be used to check if the catalyst mode is enabled
-	chain   *core.BlockChain
+// If the total difficulty was not reached:
-	state   *state.StateDB
+// 		we return INVALID
-	tcount  int
+// If the finalizedBlockHash is set:
-	gasPool *core.GasPool
+// 		we check if we have the finalizedBlockHash in our db, if not we start a sync
-
+// We try to set our blockchain to the headBlock
-	header   *types.Header
+// If there are payloadAttributes:
-	txs      []*types.Transaction
+// 		we try to assemble a block with the payloadAttributes and return its payloadID
-	receipts []*types.Receipt
+func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) {
-}
+	log.Trace("Engine API request received", "method", "ForkChoiceUpdated", "head", heads.HeadBlockHash, "finalized", heads.FinalizedBlockHash, "safe", heads.SafeBlockHash)
-
-func (env *blockExecutionEnv) commitTransaction(tx *types.Transaction, coinbase common.Address) error {
-	vmconfig := *env.chain.GetVMConfig()
-	snap := env.state.Snapshot()
-	receipt, err := core.ApplyTransaction(env.chain.Config(), env.chain, &coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vmconfig)
-	if err != nil {
-		env.state.RevertToSnapshot(snap)
-		return err
-	}
-	env.txs = append(env.txs, tx)
-	env.receipts = append(env.receipts, receipt)
-	return nil
-}
-
-func (api *ConsensusAPI) makeEnv(parent *types.Block, header *types.Header) (*blockExecutionEnv, error) {
-	// The parent state might be missing. It can be the special scenario
-	// that consensus layer tries to build a new block based on the very
-	// old side chain block and the relevant state is already pruned. So
-	// try to retrieve the live state from the chain, if it's not existent,
-	// do the necessary recovery work.
-	var (
-		err   error
-		state *state.StateDB
-	)
-	if api.eth.BlockChain().HasState(parent.Root()) {
-		state, err = api.eth.BlockChain().StateAt(parent.Root())
-	} else {
-		// The maximum acceptable reorg depth can be limited by the
-		// finalised block somehow. TODO(rjl493456442) fix the hard-
-		// coded number here later.
-		state, err = api.eth.StateAtBlock(parent, 1000, nil, false, false)
-	}
-	if err != nil {
-		return nil, err
-	}
-	env := &blockExecutionEnv{
-		chain:   api.eth.BlockChain(),
-		state:   state,
-		header:  header,
-		gasPool: new(core.GasPool).AddGas(header.GasLimit),
-	}
-	return env, nil
-}
-
-func (api *ConsensusAPI) GetPayloadV1(payloadID hexutil.Bytes) (*ExecutableDataV1, error) {
-	hash := []byte(payloadID)
-	if len(hash) < 8 {
-		return nil, &InvalidPayloadID
-	}
-	id := binary.BigEndian.Uint64(hash[:8])
-	data, ok := api.preparedBlocks[id]
-	if !ok {
-		return nil, &UnknownPayload
-	}
-	return data, nil
-}
-
-func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads ForkchoiceStateV1, PayloadAttributes *PayloadAttributesV1) (ForkChoiceResponse, error) {
 	if heads.HeadBlockHash == (common.Hash{}) {
-		return ForkChoiceResponse{Status: SUCCESS.Status, PayloadID: nil}, nil
+		return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil
 	}
 	if err := api.checkTerminalTotalDifficulty(heads.HeadBlockHash); err != nil {
 		if block := api.eth.BlockChain().GetBlockByHash(heads.HeadBlockHash); block == nil {
 			// TODO (MariusVanDerWijden) trigger sync
-			return SYNCING, nil
+			return beacon.SYNCING, nil
 		}
-		return INVALID, err
+		return beacon.INVALID, err
 	}
 	// If the finalized block is set, check if it is in our blockchain
 	if heads.FinalizedBlockHash != (common.Hash{}) {
 		if block := api.eth.BlockChain().GetBlockByHash(heads.FinalizedBlockHash); block == nil {
 			// TODO (MariusVanDerWijden) trigger sync
-			return SYNCING, nil
+			return beacon.SYNCING, nil
 		}
 	}
 	// SetHead
 	if err := api.setHead(heads.HeadBlockHash); err != nil {
-		return INVALID, err
+		return beacon.INVALID, err
 	}
-	// Assemble block (if needed)
+	// Assemble block (if needed). It only works for full node.
-	if PayloadAttributes != nil {
+	if payloadAttributes != nil {
-		data, err := api.assembleBlock(heads.HeadBlockHash, PayloadAttributes)
+		data, err := api.assembleBlock(heads.HeadBlockHash, payloadAttributes)
 		if err != nil {
-			return INVALID, err
+			return beacon.INVALID, err
 		}
-		hash := computePayloadId(heads.HeadBlockHash, PayloadAttributes)
+		id := computePayloadId(heads.HeadBlockHash, payloadAttributes)
-		id := binary.BigEndian.Uint64(hash)
+		api.preparedBlocks.put(id, data)
-		api.preparedBlocks[id] = data
+		log.Info("Created payload", "payloadID", id)
-		log.Info("Created payload", "payloadid", id)
+		return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: &id}, nil
-		// TODO (MariusVanDerWijden) do something with the payloadID?
-		hex := hexutil.Bytes(hash)
-		return ForkChoiceResponse{Status: SUCCESS.Status, PayloadID: &hex}, nil
 	}
-	return ForkChoiceResponse{Status: SUCCESS.Status, PayloadID: nil}, nil
+	return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil
 }
 
-func computePayloadId(headBlockHash common.Hash, params *PayloadAttributesV1) []byte {
+// GetPayloadV1 returns a cached payload by id.
-	// Hash
+func (api *ConsensusAPI) GetPayloadV1(payloadID beacon.PayloadID) (*beacon.ExecutableDataV1, error) {
-	hasher := sha256.New()
+	log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID)
-	hasher.Write(headBlockHash[:])
+	data := api.preparedBlocks.get(payloadID)
-	binary.Write(hasher, binary.BigEndian, params.Timestamp)
+	if data == nil {
-	hasher.Write(params.Random[:])
+		return nil, &beacon.UnknownPayload
-	hasher.Write(params.FeeRecipient[:])
-	return hasher.Sum([]byte{})[:8]
-}
-
-func (api *ConsensusAPI) invalid() ExecutePayloadResponse {
-	if api.light {
-		return ExecutePayloadResponse{Status: INVALID.Status, LatestValidHash: api.les.BlockChain().CurrentHeader().Hash()}
 	}
-	return ExecutePayloadResponse{Status: INVALID.Status, LatestValidHash: api.eth.BlockChain().CurrentHeader().Hash()}
+	return data, nil
 }
 
-// ExecutePayload creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
+// ExecutePayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
-func (api *ConsensusAPI) ExecutePayloadV1(params ExecutableDataV1) (ExecutePayloadResponse, error) {
+func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beacon.ExecutePayloadResponse, error) {
-	block, err := ExecutableDataToBlock(params)
+	log.Trace("Engine API request received", "method", "ExecutePayload", params.BlockHash, "number", params.Number)
+	block, err := beacon.ExecutableDataToBlock(params)
 	if err != nil {
 		return api.invalid(), err
 	}
-	if api.light {
-		parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash)
-		if parent == nil {
-			return api.invalid(), fmt.Errorf("could not find parent %x", params.ParentHash)
-		}
-		if err = api.les.BlockChain().InsertHeader(block.Header()); err != nil {
-			return api.invalid(), err
-		}
-		return ExecutePayloadResponse{Status: VALID.Status, LatestValidHash: block.Hash()}, nil
-	}
 	if !api.eth.BlockChain().HasBlock(block.ParentHash(), block.NumberU64()-1) {
 		/*
 			TODO (MariusVanDerWijden) reenable once sync is merged
@@ -267,7 +134,7 @@ func (api *ConsensusAPI) ExecutePayloadV1(params ExecutableDataV1) (ExecutePaylo
 			}
 		*/
 		// TODO (MariusVanDerWijden) we should return nil here not empty hash
-		return ExecutePayloadResponse{Status: SYNCING.Status, LatestValidHash: common.Hash{}}, nil
+		return beacon.ExecutePayloadResponse{Status: beacon.SYNCING.Status, LatestValidHash: common.Hash{}}, nil
 	}
 	parent := api.eth.BlockChain().GetBlockByHash(params.ParentHash)
 	td := api.eth.BlockChain().GetTd(parent.Hash(), block.NumberU64()-1)
@@ -275,188 +142,44 @@ func (api *ConsensusAPI) ExecutePayloadV1(params ExecutableDataV1) (ExecutePaylo
 	if td.Cmp(ttd) < 0 {
 		return api.invalid(), fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd)
 	}
+	log.Trace("Inserting block without head", "hash", block.Hash(), "number", block.Number)
 	if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil {
 		return api.invalid(), err
 	}
 
-	if merger := api.merger(); !merger.TDDReached() {
+	if merger := api.eth.Merger(); !merger.TDDReached() {
 		merger.ReachTTD()
 	}
-	return ExecutePayloadResponse{Status: VALID.Status, LatestValidHash: block.Hash()}, nil
+	return beacon.ExecutePayloadResponse{Status: beacon.VALID.Status, LatestValidHash: block.Hash()}, nil
 }
 
-// AssembleBlock creates a new block, inserts it into the chain, and returns the "execution
+// computePayloadId computes a pseudo-random payloadid, based on the parameters.
-// data" required for eth2 clients to process the new block.
+func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttributesV1) beacon.PayloadID {
-func (api *ConsensusAPI) assembleBlock(parentHash common.Hash, params *PayloadAttributesV1) (*ExecutableDataV1, error) {
+	// Hash
-	if api.light {
+	hasher := sha256.New()
-		return nil, errors.New("not supported")
+	hasher.Write(headBlockHash[:])
-	}
+	binary.Write(hasher, binary.BigEndian, params.Timestamp)
+	hasher.Write(params.Random[:])
+	hasher.Write(params.SuggestedFeeRecipient[:])
+	var out beacon.PayloadID
+	copy(out[:], hasher.Sum(nil)[:8])
+	return out
+}
+
+// invalid returns a response "INVALID" with the latest valid hash set to the current head.
+func (api *ConsensusAPI) invalid() beacon.ExecutePayloadResponse {
+	return beacon.ExecutePayloadResponse{Status: beacon.INVALID.Status, LatestValidHash: api.eth.BlockChain().CurrentHeader().Hash()}
+}
+
+// assembleBlock creates a new block and returns the "execution
+// data" required for beacon clients to process the new block.
+func (api *ConsensusAPI) assembleBlock(parentHash common.Hash, params *beacon.PayloadAttributesV1) (*beacon.ExecutableDataV1, error) {
 	log.Info("Producing block", "parentHash", parentHash)
-
+	block, err := api.eth.Miner().GetSealingBlock(parentHash, params.Timestamp, params.SuggestedFeeRecipient, params.Random)
-	bc := api.eth.BlockChain()
-	parent := bc.GetBlockByHash(parentHash)
-	if parent == nil {
-		log.Warn("Cannot assemble block with parent hash to unknown block", "parentHash", parentHash)
-		return nil, fmt.Errorf("cannot assemble block with unknown parent %s", parentHash)
-	}
-
-	if params.Timestamp < parent.Time() {
-		return nil, fmt.Errorf("child timestamp lower than parent's: %d < %d", params.Timestamp, parent.Time())
-	}
-	if now := uint64(time.Now().Unix()); params.Timestamp > now+1 {
-		diff := time.Duration(params.Timestamp-now) * time.Second
-		log.Warn("Producing block too far in the future", "diff", common.PrettyDuration(diff))
-	}
-	pending := api.eth.TxPool().Pending(true)
-	coinbase := params.FeeRecipient
-	num := parent.Number()
-	header := &types.Header{
-		ParentHash: parent.Hash(),
-		Number:     num.Add(num, common.Big1),
-		Coinbase:   coinbase,
-		GasLimit:   parent.GasLimit(), // Keep the gas limit constant in this prototype
-		Extra:      []byte{},          // TODO (MariusVanDerWijden) properly set extra data
-		Time:       params.Timestamp,
-	}
-	if config := api.eth.BlockChain().Config(); config.IsLondon(header.Number) {
-		header.BaseFee = misc.CalcBaseFee(config, parent.Header())
-	}
-	if err := api.engine.Prepare(bc, header); err != nil {
-		return nil, err
-	}
-	env, err := api.makeEnv(parent, header)
 	if err != nil {
 		return nil, err
 	}
-	var (
+	return beacon.BlockToExecutableData(block), nil
-		signer       = types.MakeSigner(bc.Config(), header.Number)
-		txHeap       = types.NewTransactionsByPriceAndNonce(signer, pending, nil)
-		transactions []*types.Transaction
-	)
-	for {
-		if env.gasPool.Gas() < chainParams.TxGas {
-			log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", chainParams.TxGas)
-			break
-		}
-		tx := txHeap.Peek()
-		if tx == nil {
-			break
-		}
-
-		// The sender is only for logging purposes, and it doesn't really matter if it's correct.
-		from, _ := types.Sender(signer, tx)
-
-		// Execute the transaction
-		env.state.Prepare(tx.Hash(), env.tcount)
-		err = env.commitTransaction(tx, coinbase)
-		switch err {
-		case core.ErrGasLimitReached:
-			// Pop the current out-of-gas transaction without shifting in the next from the account
-			log.Trace("Gas limit exceeded for current block", "sender", from)
-			txHeap.Pop()
-
-		case core.ErrNonceTooLow:
-			// New head notification data race between the transaction pool and miner, shift
-			log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce())
-			txHeap.Shift()
-
-		case core.ErrNonceTooHigh:
-			// Reorg notification data race between the transaction pool and miner, skip account =
-			log.Trace("Skipping account with high nonce", "sender", from, "nonce", tx.Nonce())
-			txHeap.Pop()
-
-		case nil:
-			// Everything ok, collect the logs and shift in the next transaction from the same account
-			env.tcount++
-			txHeap.Shift()
-			transactions = append(transactions, tx)
-
-		default:
-			// Strange error, discard the transaction and get the next in line (note, the
-			// nonce-too-high clause will prevent us from executing in vain).
-			log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err)
-			txHeap.Shift()
-		}
-	}
-	// Create the block.
-	block, err := api.engine.FinalizeAndAssemble(bc, header, env.state, transactions, nil /* uncles */, env.receipts)
-	if err != nil {
-		return nil, err
-	}
-	return BlockToExecutableData(block, params.Random), nil
-}
-
-func encodeTransactions(txs []*types.Transaction) [][]byte {
-	var enc = make([][]byte, len(txs))
-	for i, tx := range txs {
-		enc[i], _ = tx.MarshalBinary()
-	}
-	return enc
-}
-
-func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) {
-	var txs = make([]*types.Transaction, len(enc))
-	for i, encTx := range enc {
-		var tx types.Transaction
-		if err := tx.UnmarshalBinary(encTx); err != nil {
-			return nil, fmt.Errorf("invalid transaction %d: %v", i, err)
-		}
-		txs[i] = &tx
-	}
-	return txs, nil
-}
-
-func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) {
-	txs, err := decodeTransactions(params.Transactions)
-	if err != nil {
-		return nil, err
-	}
-	if len(params.ExtraData) > 32 {
-		return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData))
-	}
-	number := big.NewInt(0)
-	number.SetUint64(params.Number)
-	header := &types.Header{
-		ParentHash:  params.ParentHash,
-		UncleHash:   types.EmptyUncleHash,
-		Coinbase:    params.Coinbase,
-		Root:        params.StateRoot,
-		TxHash:      types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
-		ReceiptHash: params.ReceiptRoot,
-		Bloom:       types.BytesToBloom(params.LogsBloom),
-		Difficulty:  common.Big0,
-		Number:      number,
-		GasLimit:    params.GasLimit,
-		GasUsed:     params.GasUsed,
-		Time:        params.Timestamp,
-		BaseFee:     params.BaseFeePerGas,
-		Extra:       params.ExtraData,
-		// TODO (MariusVanDerWijden) add params.Random to header once required
-	}
-	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */)
-	if block.Hash() != params.BlockHash {
-		return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash())
-	}
-	return block, nil
-}
-
-func BlockToExecutableData(block *types.Block, random common.Hash) *ExecutableDataV1 {
-	return &ExecutableDataV1{
-		BlockHash:     block.Hash(),
-		ParentHash:    block.ParentHash(),
-		Coinbase:      block.Coinbase(),
-		StateRoot:     block.Root(),
-		Number:        block.NumberU64(),
-		GasLimit:      block.GasLimit(),
-		GasUsed:       block.GasUsed(),
-		BaseFeePerGas: block.BaseFee(),
-		Timestamp:     block.Time(),
-		ReceiptRoot:   block.ReceiptHash(),
-		LogsBloom:     block.Bloom().Bytes(),
-		Transactions:  encodeTransactions(block.Transactions()),
-		Random:        random,
-		ExtraData:     block.Extra(),
-	}
 }
 
 // Used in tests to add a the list of transactions from a block to the tx pool.
@@ -469,17 +192,17 @@ func (api *ConsensusAPI) insertTransactions(txs types.Transactions) error {
 
 func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error {
 	// shortcut if we entered PoS already
-	if api.merger().PoSFinalized() {
+	if api.eth.Merger().PoSFinalized() {
 		return nil
 	}
 	// make sure the parent has enough terminal total difficulty
 	newHeadBlock := api.eth.BlockChain().GetBlockByHash(head)
 	if newHeadBlock == nil {
-		return &UnknownHeader
+		return &beacon.GenericServerError
 	}
 	td := api.eth.BlockChain().GetTd(newHeadBlock.Hash(), newHeadBlock.NumberU64())
 	if td != nil && td.Cmp(api.eth.BlockChain().Config().TerminalTotalDifficulty) < 0 {
-		return errors.New("total difficulty not reached yet")
+		return &beacon.InvalidTB
 	}
 	return nil
 }
@@ -487,53 +210,22 @@ func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error {
 // setHead is called to perform a force choice.
 func (api *ConsensusAPI) setHead(newHead common.Hash) error {
 	log.Info("Setting head", "head", newHead)
-	if api.light {
-		headHeader := api.les.BlockChain().CurrentHeader()
-		if headHeader.Hash() == newHead {
-			return nil
-		}
-		newHeadHeader := api.les.BlockChain().GetHeaderByHash(newHead)
-		if newHeadHeader == nil {
-			return &UnknownHeader
-		}
-		if err := api.les.BlockChain().SetChainHead(newHeadHeader); err != nil {
-			return err
-		}
-		// Trigger the transition if it's the first `NewHead` event.
-		merger := api.merger()
-		if !merger.PoSFinalized() {
-			merger.FinalizePoS()
-		}
-		return nil
-	}
 	headBlock := api.eth.BlockChain().CurrentBlock()
 	if headBlock.Hash() == newHead {
-		// Trigger the transition if it's the first `NewHead` event.
-		if merger := api.merger(); !merger.PoSFinalized() {
-			merger.FinalizePoS()
-		}
 		return nil
 	}
 	newHeadBlock := api.eth.BlockChain().GetBlockByHash(newHead)
 	if newHeadBlock == nil {
-		return &UnknownHeader
+		return &beacon.GenericServerError
 	}
 	if err := api.eth.BlockChain().SetChainHead(newHeadBlock); err != nil {
 		return err
 	}
 	// Trigger the transition if it's the first `NewHead` event.
-	if merger := api.merger(); !merger.PoSFinalized() {
+	if merger := api.eth.Merger(); !merger.PoSFinalized() {
 		merger.FinalizePoS()
 	}
 	// TODO (MariusVanDerWijden) are we really synced now?
 	api.eth.SetSynced()
 	return nil
 }
-
-// Helper function, return the merger instance.
-func (api *ConsensusAPI) merger() *consensus.Merger {
-	if api.light {
-		return api.les.Merger()
-	}
-	return api.eth.Merger()
-}

eth/catalyst/api_test.go

@@ -17,14 +17,15 @@
 package catalyst
 
 import (
+	"fmt"
 	"math/big"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/beacon"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -78,14 +79,14 @@ func TestEth2AssembleBlock(t *testing.T) {
 	n, ethservice := startEthService(t, genesis, blocks)
 	defer n.Close()
 
-	api := NewConsensusAPI(ethservice, nil)
+	api := NewConsensusAPI(ethservice)
 	signer := types.NewEIP155Signer(ethservice.BlockChain().Config().ChainID)
 	tx, err := types.SignTx(types.NewTransaction(uint64(10), blocks[9].Coinbase(), big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, testKey)
 	if err != nil {
 		t.Fatalf("error signing transaction, err=%v", err)
 	}
 	ethservice.TxPool().AddLocal(tx)
-	blockParams := PayloadAttributesV1{
+	blockParams := beacon.PayloadAttributesV1{
 		Timestamp: blocks[9].Time() + 5,
 	}
 	execData, err := api.assembleBlock(blocks[9].Hash(), &blockParams)
@@ -102,11 +103,11 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) {
 	n, ethservice := startEthService(t, genesis, blocks[:9])
 	defer n.Close()
 
-	api := NewConsensusAPI(ethservice, nil)
+	api := NewConsensusAPI(ethservice)
 
 	// Put the 10th block's tx in the pool and produce a new block
 	api.insertTransactions(blocks[9].Transactions())
-	blockParams := PayloadAttributesV1{
+	blockParams := beacon.PayloadAttributesV1{
 		Timestamp: blocks[8].Time() + 5,
 	}
 	execData, err := api.assembleBlock(blocks[8].Hash(), &blockParams)
@@ -123,8 +124,8 @@ func TestSetHeadBeforeTotalDifficulty(t *testing.T) {
 	n, ethservice := startEthService(t, genesis, blocks)
 	defer n.Close()
 
-	api := NewConsensusAPI(ethservice, nil)
+	api := NewConsensusAPI(ethservice)
-	fcState := ForkchoiceStateV1{
+	fcState := beacon.ForkchoiceStateV1{
 		HeadBlockHash:      blocks[5].Hash(),
 		SafeBlockHash:      common.Hash{},
 		FinalizedBlockHash: common.Hash{},
@@ -141,14 +142,14 @@ func TestEth2PrepareAndGetPayload(t *testing.T) {
 	n, ethservice := startEthService(t, genesis, blocks[:9])
 	defer n.Close()
 
-	api := NewConsensusAPI(ethservice, nil)
+	api := NewConsensusAPI(ethservice)
 
 	// Put the 10th block's tx in the pool and produce a new block
 	api.insertTransactions(blocks[9].Transactions())
-	blockParams := PayloadAttributesV1{
+	blockParams := beacon.PayloadAttributesV1{
 		Timestamp: blocks[8].Time() + 5,
 	}
-	fcState := ForkchoiceStateV1{
+	fcState := beacon.ForkchoiceStateV1{
 		HeadBlockHash:      blocks[8].Hash(),
 		SafeBlockHash:      common.Hash{},
 		FinalizedBlockHash: common.Hash{},
@@ -158,13 +159,21 @@ func TestEth2PrepareAndGetPayload(t *testing.T) {
 		t.Fatalf("error preparing payload, err=%v", err)
 	}
 	payloadID := computePayloadId(fcState.HeadBlockHash, &blockParams)
-	execData, err := api.GetPayloadV1(hexutil.Bytes(payloadID))
+	execData, err := api.GetPayloadV1(payloadID)
 	if err != nil {
 		t.Fatalf("error getting payload, err=%v", err)
 	}
 	if len(execData.Transactions) != blocks[9].Transactions().Len() {
 		t.Fatalf("invalid number of transactions %d != 1", len(execData.Transactions))
 	}
+	// Test invalid payloadID
+	var invPayload beacon.PayloadID
+	copy(invPayload[:], payloadID[:])
+	invPayload[0] = ^invPayload[0]
+	_, err = api.GetPayloadV1(invPayload)
+	if err == nil {
+		t.Fatal("expected error retrieving invalid payload")
+	}
 }
 
 func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan core.RemovedLogsEvent, wantNew, wantRemoved int) {
@@ -185,6 +194,48 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan co
 	}
 }
 
+func TestInvalidPayloadTimestamp(t *testing.T) {
+	genesis, preMergeBlocks := generatePreMergeChain(10)
+	n, ethservice := startEthService(t, genesis, preMergeBlocks)
+	ethservice.Merger().ReachTTD()
+	defer n.Close()
+	var (
+		api    = NewConsensusAPI(ethservice)
+		parent = ethservice.BlockChain().CurrentBlock()
+	)
+	tests := []struct {
+		time      uint64
+		shouldErr bool
+	}{
+		{0, true},
+		{parent.Time(), true},
+		{parent.Time() - 1, true},
+		{parent.Time() + 1, false},
+		{uint64(time.Now().Unix()) + uint64(time.Minute), false},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Timestamp test: %v", i), func(t *testing.T) {
+			params := beacon.PayloadAttributesV1{
+				Timestamp:             test.time,
+				Random:                crypto.Keccak256Hash([]byte{byte(123)}),
+				SuggestedFeeRecipient: parent.Coinbase(),
+			}
+			fcState := beacon.ForkchoiceStateV1{
+				HeadBlockHash:      parent.Hash(),
+				SafeBlockHash:      common.Hash{},
+				FinalizedBlockHash: common.Hash{},
+			}
+			_, err := api.ForkchoiceUpdatedV1(fcState, &params)
+			if test.shouldErr && err == nil {
+				t.Fatalf("expected error preparing payload with invalid timestamp, err=%v", err)
+			} else if !test.shouldErr && err != nil {
+				t.Fatalf("error preparing payload with valid timestamp, err=%v", err)
+			}
+		})
+	}
+}
+
 func TestEth2NewBlock(t *testing.T) {
 	genesis, preMergeBlocks := generatePreMergeChain(10)
 	n, ethservice := startEthService(t, genesis, preMergeBlocks)
@@ -192,7 +243,7 @@ func TestEth2NewBlock(t *testing.T) {
 	defer n.Close()
 
 	var (
-		api    = NewConsensusAPI(ethservice, nil)
+		api    = NewConsensusAPI(ethservice)
 		parent = preMergeBlocks[len(preMergeBlocks)-1]
 
 		// This EVM code generates a log when the contract is created.
@@ -210,13 +261,13 @@ func TestEth2NewBlock(t *testing.T) {
 		tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey)
 		ethservice.TxPool().AddLocal(tx)
 
-		execData, err := api.assembleBlock(parent.Hash(), &PayloadAttributesV1{
+		execData, err := api.assembleBlock(parent.Hash(), &beacon.PayloadAttributesV1{
 			Timestamp: parent.Time() + 5,
 		})
 		if err != nil {
 			t.Fatalf("Failed to create the executable data %v", err)
 		}
-		block, err := ExecutableDataToBlock(*execData)
+		block, err := beacon.ExecutableDataToBlock(*execData)
 		if err != nil {
 			t.Fatalf("Failed to convert executable data to block %v", err)
 		}
@@ -228,7 +279,7 @@ func TestEth2NewBlock(t *testing.T) {
 			t.Fatalf("Chain head shouldn't be updated")
 		}
 		checkLogEvents(t, newLogCh, rmLogsCh, 0, 0)
-		fcState := ForkchoiceStateV1{
+		fcState := beacon.ForkchoiceStateV1{
 			HeadBlockHash:      block.Hash(),
 			SafeBlockHash:      block.Hash(),
 			FinalizedBlockHash: block.Hash(),
@@ -250,13 +301,13 @@ func TestEth2NewBlock(t *testing.T) {
 	)
 	parent = preMergeBlocks[len(preMergeBlocks)-1]
 	for i := 0; i < 10; i++ {
-		execData, err := api.assembleBlock(parent.Hash(), &PayloadAttributesV1{
+		execData, err := api.assembleBlock(parent.Hash(), &beacon.PayloadAttributesV1{
 			Timestamp: parent.Time() + 6,
 		})
 		if err != nil {
 			t.Fatalf("Failed to create the executable data %v", err)
 		}
-		block, err := ExecutableDataToBlock(*execData)
+		block, err := beacon.ExecutableDataToBlock(*execData)
 		if err != nil {
 			t.Fatalf("Failed to convert executable data to block %v", err)
 		}
@@ -268,7 +319,7 @@ func TestEth2NewBlock(t *testing.T) {
 			t.Fatalf("Chain head shouldn't be updated")
 		}
 
-		fcState := ForkchoiceStateV1{
+		fcState := beacon.ForkchoiceStateV1{
 			HeadBlockHash:      block.Hash(),
 			SafeBlockHash:      block.Hash(),
 			FinalizedBlockHash: block.Hash(),
@@ -362,7 +413,7 @@ func TestFullAPI(t *testing.T) {
 	ethservice.Merger().ReachTTD()
 	defer n.Close()
 	var (
-		api    = NewConsensusAPI(ethservice, nil)
+		api    = NewConsensusAPI(ethservice)
 		parent = ethservice.BlockChain().CurrentBlock()
 		// This EVM code generates a log when the contract is created.
 		logCode = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00")
@@ -373,12 +424,12 @@ func TestFullAPI(t *testing.T) {
 		tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey)
 		ethservice.TxPool().AddLocal(tx)
 
-		params := PayloadAttributesV1{
+		params := beacon.PayloadAttributesV1{
-			Timestamp:    parent.Time() + 1,
+			Timestamp:             parent.Time() + 1,
-			Random:       crypto.Keccak256Hash([]byte{byte(i)}),
+			Random:                crypto.Keccak256Hash([]byte{byte(i)}),
-			FeeRecipient: parent.Coinbase(),
+			SuggestedFeeRecipient: parent.Coinbase(),
 		}
-		fcState := ForkchoiceStateV1{
+		fcState := beacon.ForkchoiceStateV1{
 			HeadBlockHash:      parent.Hash(),
 			SafeBlockHash:      common.Hash{},
 			FinalizedBlockHash: common.Hash{},
@@ -387,11 +438,11 @@ func TestFullAPI(t *testing.T) {
 		if err != nil {
 			t.Fatalf("error preparing payload, err=%v", err)
 		}
-		if resp.Status != SUCCESS.Status {
+		if resp.Status != beacon.SUCCESS.Status {
 			t.Fatalf("error preparing payload, invalid status: %v", resp.Status)
 		}
 		payloadID := computePayloadId(parent.Hash(), &params)
-		payload, err := api.GetPayloadV1(hexutil.Bytes(payloadID))
+		payload, err := api.GetPayloadV1(payloadID)
 		if err != nil {
 			t.Fatalf("can't get payload: %v", err)
 		}
@@ -399,10 +450,10 @@ func TestFullAPI(t *testing.T) {
 		if err != nil {
 			t.Fatalf("can't execute payload: %v", err)
 		}
-		if execResp.Status != VALID.Status {
+		if execResp.Status != beacon.VALID.Status {
 			t.Fatalf("invalid status: %v", execResp.Status)
 		}
-		fcState = ForkchoiceStateV1{
+		fcState = beacon.ForkchoiceStateV1{
 			HeadBlockHash:      payload.BlockHash,
 			SafeBlockHash:      payload.ParentHash,
 			FinalizedBlockHash: payload.ParentHash,
@@ -414,6 +465,5 @@ func TestFullAPI(t *testing.T) {
 			t.Fatalf("Chain head should be updated")
 		}
 		parent = ethservice.BlockChain().CurrentBlock()
-
 	}
 }

eth/catalyst/api_types.go

@@ -1,114 +0,0 @@
-// Copyright 2020 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package catalyst
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-)
-
-//go:generate go run github.com/fjl/gencodec -type PayloadAttributesV1 -field-override payloadAttributesMarshaling -out gen_blockparams.go
-
-// Structure described at https://github.com/ethereum/execution-apis/pull/74
-type PayloadAttributesV1 struct {
-	Timestamp    uint64         `json:"timestamp"     gencodec:"required"`
-	Random       common.Hash    `json:"random"        gencodec:"required"`
-	FeeRecipient common.Address `json:"feeRecipient"  gencodec:"required"`
-}
-
-// JSON type overrides for PayloadAttributesV1.
-type payloadAttributesMarshaling struct {
-	Timestamp hexutil.Uint64
-}
-
-//go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go
-
-// Structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md
-type ExecutableDataV1 struct {
-	ParentHash    common.Hash    `json:"parentHash"    gencodec:"required"`
-	Coinbase      common.Address `json:"coinbase"      gencodec:"required"`
-	StateRoot     common.Hash    `json:"stateRoot"     gencodec:"required"`
-	ReceiptRoot   common.Hash    `json:"receiptRoot"   gencodec:"required"`
-	LogsBloom     []byte         `json:"logsBloom"     gencodec:"required"`
-	Random        common.Hash    `json:"random"        gencodec:"required"`
-	Number        uint64         `json:"blockNumber"   gencodec:"required"`
-	GasLimit      uint64         `json:"gasLimit"      gencodec:"required"`
-	GasUsed       uint64         `json:"gasUsed"       gencodec:"required"`
-	Timestamp     uint64         `json:"timestamp"     gencodec:"required"`
-	ExtraData     []byte         `json:"extraData"     gencodec:"required"`
-	BaseFeePerGas *big.Int       `json:"baseFeePerGas" gencodec:"required"`
-	BlockHash     common.Hash    `json:"blockHash"     gencodec:"required"`
-	Transactions  [][]byte       `json:"transactions"  gencodec:"required"`
-}
-
-// JSON type overrides for executableData.
-type executableDataMarshaling struct {
-	Number        hexutil.Uint64
-	GasLimit      hexutil.Uint64
-	GasUsed       hexutil.Uint64
-	Timestamp     hexutil.Uint64
-	BaseFeePerGas *hexutil.Big
-	ExtraData     hexutil.Bytes
-	LogsBloom     hexutil.Bytes
-	Transactions  []hexutil.Bytes
-}
-
-//go:generate go run github.com/fjl/gencodec -type PayloadResponse -field-override payloadResponseMarshaling -out gen_payload.go
-
-type PayloadResponse struct {
-	PayloadID uint64 `json:"payloadId"`
-}
-
-// JSON type overrides for payloadResponse.
-type payloadResponseMarshaling struct {
-	PayloadID hexutil.Uint64
-}
-
-type NewBlockResponse struct {
-	Valid bool `json:"valid"`
-}
-
-type GenericResponse struct {
-	Success bool `json:"success"`
-}
-
-type GenericStringResponse struct {
-	Status string `json:"status"`
-}
-
-type ExecutePayloadResponse struct {
-	Status          string      `json:"status"`
-	LatestValidHash common.Hash `json:"latestValidHash"`
-}
-
-type ConsensusValidatedParams struct {
-	BlockHash common.Hash `json:"blockHash"`
-	Status    string      `json:"status"`
-}
-
-type ForkChoiceResponse struct {
-	Status    string         `json:"status"`
-	PayloadID *hexutil.Bytes `json:"payloadId"`
-}
-
-type ForkchoiceStateV1 struct {
-	HeadBlockHash      common.Hash `json:"headBlockHash"`
-	SafeBlockHash      common.Hash `json:"safeBlockHash"`
-	FinalizedBlockHash common.Hash `json:"finalizedBlockHash"`
-}

eth/catalyst/gen_payload.go

@@ -1,36 +0,0 @@
-// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
-
-package catalyst
-
-import (
-	"encoding/json"
-
-	"github.com/ethereum/go-ethereum/common/hexutil"
-)
-
-var _ = (*payloadResponseMarshaling)(nil)
-
-// MarshalJSON marshals as JSON.
-func (p PayloadResponse) MarshalJSON() ([]byte, error) {
-	type PayloadResponse struct {
-		PayloadID hexutil.Uint64 `json:"payloadId"`
-	}
-	var enc PayloadResponse
-	enc.PayloadID = hexutil.Uint64(p.PayloadID)
-	return json.Marshal(&enc)
-}
-
-// UnmarshalJSON unmarshals from JSON.
-func (p *PayloadResponse) UnmarshalJSON(input []byte) error {
-	type PayloadResponse struct {
-		PayloadID *hexutil.Uint64 `json:"payloadId"`
-	}
-	var dec PayloadResponse
-	if err := json.Unmarshal(input, &dec); err != nil {
-		return err
-	}
-	if dec.PayloadID != nil {
-		p.PayloadID = uint64(*dec.PayloadID)
-	}
-	return nil
-}

eth/catalyst/queue.go

@@ -0,0 +1,78 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package catalyst
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/core/beacon"
+)
+
+// maxTrackedPayloads is the maximum number of prepared payloads the execution
+// engine tracks before evicting old ones. Ideally we should only ever track the
+// latest one; but have a slight wiggle room for non-ideal conditions.
+const maxTrackedPayloads = 10
+
+// payloadQueueItem represents an id->payload tuple to store until it's retrieved
+// or evicted.
+type payloadQueueItem struct {
+	id      beacon.PayloadID
+	payload *beacon.ExecutableDataV1
+}
+
+// payloadQueue tracks the latest handful of constructed payloads to be retrieved
+// by the beacon chain if block production is requested.
+type payloadQueue struct {
+	payloads []*payloadQueueItem
+	lock     sync.RWMutex
+}
+
+// newPayloadQueue creates a pre-initialized queue with a fixed number of slots
+// all containing empty items.
+func newPayloadQueue() *payloadQueue {
+	return &payloadQueue{
+		payloads: make([]*payloadQueueItem, maxTrackedPayloads),
+	}
+}
+
+// put inserts a new payload into the queue at the given id.
+func (q *payloadQueue) put(id beacon.PayloadID, data *beacon.ExecutableDataV1) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	copy(q.payloads[1:], q.payloads)
+	q.payloads[0] = &payloadQueueItem{
+		id:      id,
+		payload: data,
+	}
+}
+
+// get retrieves a previously stored payload item or nil if it does not exist.
+func (q *payloadQueue) get(id beacon.PayloadID) *beacon.ExecutableDataV1 {
+	q.lock.RLock()
+	defer q.lock.RUnlock()
+
+	for _, item := range q.payloads {
+		if item == nil {
+			return nil // no more items
+		}
+		if item.id == id {
+			return item.payload
+		}
+	}
+	return nil
+}

eth/fetcher/block_fetcher_test.go

@@ -364,6 +364,7 @@ func testSequentialAnnouncements(t *testing.T, light bool) {
 	hashes, blocks := makeChain(targetBlocks, 0, genesis)
 
 	tester := newTester(light)
+	defer tester.fetcher.Stop()
 	headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack)
 	bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0)
 
@@ -743,7 +744,7 @@ func testInvalidNumberAnnouncement(t *testing.T, light bool) {
 	badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0)
 
 	imported := make(chan interface{})
-	announced := make(chan interface{})
+	announced := make(chan interface{}, 2)
 	tester.fetcher.importedHook = func(header *types.Header, block *types.Block) {
 		if light {
 			if header == nil {
@@ -806,6 +807,7 @@ func TestEmptyBlockShortCircuit(t *testing.T) {
 	hashes, blocks := makeChain(32, 0, genesis)
 
 	tester := newTester(false)
+	defer tester.fetcher.Stop()
 	headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack)
 	bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0)
 

eth/handler.go

@@ -433,7 +433,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error {
 					return
 				}
 				peer.Log().Debug("Whitelist block verified", "number", number, "hash", hash)
-
+				res.Done <- nil
 			case <-timeout.C:
 				peer.Log().Warn("Whitelist challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name())
 				h.removePeer(peer.ID())

eth/protocols/snap/sync.go

@@ -1781,7 +1781,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) {
 	for i, account := range res.accounts {
 		// Check if the account is a contract with an unknown code
 		if !bytes.Equal(account.CodeHash, emptyCode[:]) {
-			if code := rawdb.ReadCodeWithPrefix(s.db, common.BytesToHash(account.CodeHash)); code == nil {
+			if !rawdb.HasCodeWithPrefix(s.db, common.BytesToHash(account.CodeHash)) {
 				res.task.codeTasks[common.BytesToHash(account.CodeHash)] = struct{}{}
 				res.task.needCode[i] = true
 				res.task.pend++
@@ -1789,7 +1789,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) {
 		}
 		// Check if the account is a contract with an unknown storage trie
 		if account.Root != emptyRoot {
-			if node, err := s.db.Get(account.Root[:]); err != nil || node == nil {
+			if ok, err := s.db.Has(account.Root[:]); err != nil || !ok {
 				// If there was a previous large state retrieval in progress,
 				// don't restart it from scratch. This happens if a sync cycle
 				// is interrupted and resumed later. However, *do* update the

eth/tracers/api.go

@@ -592,11 +592,11 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac
 	if threads > len(txs) {
 		threads = len(txs)
 	}
-	blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil)
 	blockHash := block.Hash()
 	for th := 0; th < threads; th++ {
 		pend.Add(1)
 		go func() {
+			blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil)
 			defer pend.Done()
 			// Fetch and execute the next transaction trace tasks
 			for task := range jobs {
@@ -617,6 +617,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac
 	}
 	// Feed the transactions into the tracers and return
 	var failed error
+	blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil)
 	for i, tx := range txs {
 		// Send the trace task over for execution
 		jobs <- &txTraceTask{statedb: statedb.Copy(), index: i}

eth/tracers/internal/tracetest/calltrace_test.go

@@ -130,10 +130,6 @@ func TestCallTracerLegacy(t *testing.T) {
 	testCallTracer("callTracerLegacy", "call_tracer_legacy", t)
 }
 
-func TestCallTracerJs(t *testing.T) {
-	testCallTracer("callTracerJs", "call_tracer", t)
-}
-
 func TestCallTracerNative(t *testing.T) {
 	testCallTracer("callTracer", "call_tracer", t)
 }

eth/tracers/internal/tracetest/testdata/call_tracer/inner_throw_outer_revert.json

@@ -59,7 +59,7 @@
   "result": {
     "calls": [
       {
-        "error": "invalid opcode: opcode 0xfe not defined",
+        "error": "invalid opcode: INVALID",
         "from": "0x33056b5dcac09a9b4becad0e1dcf92c19bd0af76",
         "gas": "0x75fe3",
         "gasUsed": "0x75fe3",

eth/tracers/internal/tracetest/testdata/call_tracer_legacy/inner_throw_outer_revert.json

@@ -59,7 +59,7 @@
   "result": {
     "calls": [
       {
-        "error": "invalid opcode: opcode 0xfe not defined",
+        "error": "invalid opcode: INVALID",
         "from": "0x33056b5dcac09a9b4becad0e1dcf92c19bd0af76",
         "gas": "0x75fe3",
         "gasUsed": "0x75fe3",

eth/tracers/js/internal/tracers/call_tracer_js.js

@@ -1,112 +0,0 @@
-// Copyright 2021 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-
-// callFrameTracer uses the new call frame tracing methods to report useful information
-// about internal messages of a transaction.
-{
-	callstack: [{}],
-	fault: function(log, db) {},
-	result: function(ctx, db) {
-		// Prepare outer message info
-		var result = {
-			type:    ctx.type,
-			from:    toHex(ctx.from),
-			to:      toHex(ctx.to),
-			value:   '0x' + ctx.value.toString(16),
-			gas:     '0x' + bigInt(ctx.gas).toString(16),
-			gasUsed: '0x' + bigInt(ctx.gasUsed).toString(16),
-			input:   toHex(ctx.input),
-			output:  toHex(ctx.output),
-		}
-		if (this.callstack[0].calls !== undefined) {
-			result.calls = this.callstack[0].calls
-		}
-		if (this.callstack[0].error !== undefined) {
-			result.error = this.callstack[0].error
-		} else if (ctx.error !== undefined) {
-			result.error = ctx.error
-		}
-		if (result.error !== undefined && (result.error !== "execution reverted" || result.output ==="0x")) {
-			delete result.output
-		}
-
-		return this.finalize(result)
-	},
-	enter: function(frame) {
-		var call = {
-			type: frame.getType(),
-			from: toHex(frame.getFrom()),
-			to: toHex(frame.getTo()),
-			input: toHex(frame.getInput()),
-			gas: '0x' + bigInt(frame.getGas()).toString('16'),
-		}
-		if (frame.getValue() !== undefined){
-			call.value='0x' + bigInt(frame.getValue()).toString(16)
-		}
-		this.callstack.push(call)
-	},
-	exit: function(frameResult) {
-		var len = this.callstack.length
-		if (len > 1) {
-			var call = this.callstack.pop()
-			call.gasUsed = '0x' + bigInt(frameResult.getGasUsed()).toString('16')
-			var error = frameResult.getError()
-			if (error === undefined) {
-				call.output = toHex(frameResult.getOutput())
-			} else {
-				call.error = error
-				if (call.type === 'CREATE' || call.type === 'CREATE2') {
-					delete call.to
-				}
-			}
-			len -= 1
-			if (this.callstack[len-1].calls === undefined) {
-				this.callstack[len-1].calls = []
-			}
-			this.callstack[len-1].calls.push(call)
-		}
-	},
-	// finalize recreates a call object using the final desired field oder for json
-	// serialization. This is a nicety feature to pass meaningfully ordered results
-	// to users who don't interpret it, just display it.
-	finalize: function(call) {
-		var sorted = {
-			type:    call.type,
-			from:    call.from,
-			to:      call.to,
-			value:   call.value,
-			gas:     call.gas,
-			gasUsed: call.gasUsed,
-			input:   call.input,
-			output:  call.output,
-			error:   call.error,
-			time:    call.time,
-			calls:   call.calls,
-		}
-		for (var key in sorted) {
-			if (sorted[key] === undefined) {
-				delete sorted[key]
-			}
-		}
-		if (sorted.calls !== undefined) {
-			for (var i=0; i<sorted.calls.length; i++) {
-				sorted.calls[i] = this.finalize(sorted.calls[i])
-			}
-		}
-		return sorted
-	}
-}

eth/tracers/js/internal/tracers/prestate_tracer_legacy.js

@@ -47,6 +47,13 @@
 	// result is invoked when all the opcodes have been iterated over and returns
 	// the final result of the tracing.
 	result: function(ctx, db) {
+		if (this.prestate === null) {
+			this.prestate = {};
+			// If tx is transfer-only, the recipient account
+			// hasn't been populated.
+			this.lookupAccount(ctx.to, db);
+		}
+
 		// At this point, we need to deduct the 'value' from the
 		// outer transaction, and move it back to the origin
 		this.lookupAccount(ctx.from, db);
@@ -79,7 +86,7 @@
 		}
 		// Whenever new state is accessed, add it to the prestate
 		switch (log.op.toString()) {
-			case "EXTCODECOPY": case "EXTCODESIZE": case "BALANCE":
+			case "EXTCODECOPY": case "EXTCODESIZE": case "EXTCODEHASH": case "BALANCE":
 				this.lookupAccount(toAddress(log.stack.peek(0).toString(16)), db);
 				break;
 			case "CREATE":

eth/tracers/js/tracer.go

@@ -697,7 +697,7 @@ func (jst *jsTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Ad
 	jst.ctx["block"] = env.Context.BlockNumber.Uint64()
 	jst.dbWrapper.db = env.StateDB
 	// Update list of precompiles based on current block
-	rules := env.ChainConfig().Rules(env.Context.BlockNumber)
+	rules := env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil)
 	jst.activePrecompiles = vm.ActivePrecompiles(rules)
 
 	// Compute intrinsic gas

eth/tracers/native/4byte.go

@@ -83,7 +83,7 @@ func (t *fourByteTracer) CaptureStart(env *vm.EVM, from common.Address, to commo
 	t.env = env
 
 	// Update list of precompiles based on current block
-	rules := env.ChainConfig().Rules(env.Context.BlockNumber)
+	rules := env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Random != nil)
 	t.activePrecompiles = vm.ActivePrecompiles(rules)
 
 	// Save the outer calldata also

eth/tracers/native/call.go

@@ -59,8 +59,7 @@ type callTracer struct {
 func newCallTracer() tracers.Tracer {
 	// First callframe contains tx context info
 	// and is populated on start and end.
-	t := &callTracer{callstack: make([]callFrame, 1)}
+	return &callTracer{callstack: make([]callFrame, 1)}
-	return t
 }
 
 // CaptureStart implements the EVMLogger interface to initialize the tracing operation.

eth/tracers/native/noop.go

@@ -27,7 +27,7 @@ import (
 )
 
 func init() {
-	register("noopTracerNative", newNoopTracer)
+	register("noopTracer", newNoopTracer)
 }
 
 // noopTracer is a go implementation of the Tracer interface which

eth/tracers/native/prestate.go

@@ -0,0 +1,186 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package native
+
+import (
+	"encoding/json"
+	"math/big"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+)
+
+func init() {
+	register("prestateTracer", newPrestateTracer)
+}
+
+type prestate = map[common.Address]*account
+type account struct {
+	Balance string                      `json:"balance"`
+	Nonce   uint64                      `json:"nonce"`
+	Code    string                      `json:"code"`
+	Storage map[common.Hash]common.Hash `json:"storage"`
+}
+
+type prestateTracer struct {
+	env       *vm.EVM
+	prestate  prestate
+	create    bool
+	to        common.Address
+	interrupt uint32 // Atomic flag to signal execution interruption
+	reason    error  // Textual reason for the interruption
+}
+
+func newPrestateTracer() tracers.Tracer {
+	// First callframe contains tx context info
+	// and is populated on start and end.
+	return &prestateTracer{prestate: prestate{}}
+}
+
+// CaptureStart implements the EVMLogger interface to initialize the tracing operation.
+func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) {
+	t.env = env
+	t.create = create
+	t.to = to
+
+	// Compute intrinsic gas
+	isHomestead := env.ChainConfig().IsHomestead(env.Context.BlockNumber)
+	isIstanbul := env.ChainConfig().IsIstanbul(env.Context.BlockNumber)
+	intrinsicGas, err := core.IntrinsicGas(input, nil, create, isHomestead, isIstanbul)
+	if err != nil {
+		return
+	}
+
+	t.lookupAccount(from)
+	t.lookupAccount(to)
+
+	// The recipient balance includes the value transferred.
+	toBal := hexutil.MustDecodeBig(t.prestate[to].Balance)
+	toBal = new(big.Int).Sub(toBal, value)
+	t.prestate[to].Balance = hexutil.EncodeBig(toBal)
+
+	// The sender balance is after reducing: value, gasLimit, intrinsicGas.
+	// We need to re-add them to get the pre-tx balance.
+	fromBal := hexutil.MustDecodeBig(t.prestate[from].Balance)
+	gasPrice := env.TxContext.GasPrice
+	consumedGas := new(big.Int).Mul(
+		gasPrice,
+		new(big.Int).Add(
+			new(big.Int).SetUint64(intrinsicGas),
+			new(big.Int).SetUint64(gas),
+		),
+	)
+	fromBal.Add(fromBal, new(big.Int).Add(value, consumedGas))
+	t.prestate[from].Balance = hexutil.EncodeBig(fromBal)
+	t.prestate[from].Nonce--
+}
+
+// CaptureEnd is called after the call finishes to finalize the tracing.
+func (t *prestateTracer) CaptureEnd(output []byte, gasUsed uint64, _ time.Duration, err error) {
+	if t.create {
+		// Exclude created contract.
+		delete(t.prestate, t.to)
+	}
+}
+
+// CaptureState implements the EVMLogger interface to trace a single step of VM execution.
+func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) {
+	stack := scope.Stack
+	stackData := stack.Data()
+	stackLen := len(stackData)
+	switch {
+	case stackLen >= 1 && (op == vm.SLOAD || op == vm.SSTORE):
+		slot := common.Hash(stackData[stackLen-1].Bytes32())
+		t.lookupStorage(scope.Contract.Address(), slot)
+	case stackLen >= 1 && (op == vm.EXTCODECOPY || op == vm.EXTCODEHASH || op == vm.EXTCODESIZE || op == vm.BALANCE || op == vm.SELFDESTRUCT):
+		addr := common.Address(stackData[stackLen-1].Bytes20())
+		t.lookupAccount(addr)
+	case stackLen >= 5 && (op == vm.DELEGATECALL || op == vm.CALL || op == vm.STATICCALL || op == vm.CALLCODE):
+		addr := common.Address(stackData[stackLen-2].Bytes20())
+		t.lookupAccount(addr)
+	case op == vm.CREATE:
+		addr := scope.Contract.Address()
+		nonce := t.env.StateDB.GetNonce(addr)
+		t.lookupAccount(crypto.CreateAddress(addr, nonce))
+	case stackLen >= 4 && op == vm.CREATE2:
+		offset := stackData[stackLen-2]
+		size := stackData[stackLen-3]
+		init := scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64()))
+		inithash := crypto.Keccak256(init)
+		salt := stackData[stackLen-4]
+		t.lookupAccount(crypto.CreateAddress2(scope.Contract.Address(), salt.Bytes32(), inithash))
+	}
+}
+
+// CaptureFault implements the EVMLogger interface to trace an execution fault.
+func (t *prestateTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) {
+}
+
+// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct).
+func (t *prestateTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) {
+}
+
+// CaptureExit is called when EVM exits a scope, even if the scope didn't
+// execute any code.
+func (t *prestateTracer) CaptureExit(output []byte, gasUsed uint64, err error) {
+}
+
+// GetResult returns the json-encoded nested list of call traces, and any
+// error arising from the encoding or forceful termination (via `Stop`).
+func (t *prestateTracer) GetResult() (json.RawMessage, error) {
+	res, err := json.Marshal(t.prestate)
+	if err != nil {
+		return nil, err
+	}
+	return json.RawMessage(res), t.reason
+}
+
+// Stop terminates execution of the tracer at the first opportune moment.
+func (t *prestateTracer) Stop(err error) {
+	t.reason = err
+	atomic.StoreUint32(&t.interrupt, 1)
+}
+
+// lookupAccount fetches details of an account and adds it to the prestate
+// if it doesn't exist there.
+func (t *prestateTracer) lookupAccount(addr common.Address) {
+	if _, ok := t.prestate[addr]; ok {
+		return
+	}
+	t.prestate[addr] = &account{
+		Balance: bigToHex(t.env.StateDB.GetBalance(addr)),
+		Nonce:   t.env.StateDB.GetNonce(addr),
+		Code:    bytesToHex(t.env.StateDB.GetCode(addr)),
+		Storage: make(map[common.Hash]common.Hash),
+	}
+}
+
+// lookupStorage fetches the requested storage slot and adds
+// it to the prestate of the given contract. It assumes `lookupAccount`
+// has been performed on the contract before.
+func (t *prestateTracer) lookupStorage(addr common.Address, key common.Hash) {
+	if _, ok := t.prestate[addr].Storage[key]; ok {
+		return
+	}
+	t.prestate[addr].Storage[key] = t.env.StateDB.GetState(addr, key)
+}

ethclient/ethclient.go

@@ -298,11 +298,11 @@ func (ec *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, err
 	if err := json.Unmarshal(raw, &syncing); err == nil {
 		return nil, nil // Not syncing (always false)
 	}
-	var progress *ethereum.SyncProgress
+	var p *rpcProgress
-	if err := json.Unmarshal(raw, &progress); err != nil {
+	if err := json.Unmarshal(raw, &p); err != nil {
 		return nil, err
 	}
-	return progress, nil
+	return p.toSyncProgress(), nil
 }
 
 // SubscribeNewHead subscribes to notifications about the current blockchain head
@@ -542,3 +542,51 @@ func toCallArg(msg ethereum.CallMsg) interface{} {
 	}
 	return arg
 }
+
+// rpcProgress is a copy of SyncProgress with hex-encoded fields.
+type rpcProgress struct {
+	StartingBlock hexutil.Uint64
+	CurrentBlock  hexutil.Uint64
+	HighestBlock  hexutil.Uint64
+
+	PulledStates hexutil.Uint64
+	KnownStates  hexutil.Uint64
+
+	SyncedAccounts      hexutil.Uint64
+	SyncedAccountBytes  hexutil.Uint64
+	SyncedBytecodes     hexutil.Uint64
+	SyncedBytecodeBytes hexutil.Uint64
+	SyncedStorage       hexutil.Uint64
+	SyncedStorageBytes  hexutil.Uint64
+	HealedTrienodes     hexutil.Uint64
+	HealedTrienodeBytes hexutil.Uint64
+	HealedBytecodes     hexutil.Uint64
+	HealedBytecodeBytes hexutil.Uint64
+	HealingTrienodes    hexutil.Uint64
+	HealingBytecode     hexutil.Uint64
+}
+
+func (p *rpcProgress) toSyncProgress() *ethereum.SyncProgress {
+	if p == nil {
+		return nil
+	}
+	return &ethereum.SyncProgress{
+		StartingBlock:       uint64(p.StartingBlock),
+		CurrentBlock:        uint64(p.CurrentBlock),
+		HighestBlock:        uint64(p.HighestBlock),
+		PulledStates:        uint64(p.PulledStates),
+		KnownStates:         uint64(p.KnownStates),
+		SyncedAccounts:      uint64(p.SyncedAccounts),
+		SyncedAccountBytes:  uint64(p.SyncedAccountBytes),
+		SyncedBytecodes:     uint64(p.SyncedBytecodes),
+		SyncedBytecodeBytes: uint64(p.SyncedBytecodeBytes),
+		SyncedStorage:       uint64(p.SyncedStorage),
+		SyncedStorageBytes:  uint64(p.SyncedStorageBytes),
+		HealedTrienodes:     uint64(p.HealedTrienodes),
+		HealedTrienodeBytes: uint64(p.HealedTrienodeBytes),
+		HealedBytecodes:     uint64(p.HealedBytecodes),
+		HealedBytecodeBytes: uint64(p.HealedBytecodeBytes),
+		HealingTrienodes:    uint64(p.HealingTrienodes),
+		HealingBytecode:     uint64(p.HealingBytecode),
+	}
+}

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/cloudflare/cloudflare-go v0.14.0
 	github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f
 	github.com/davecgh/go-spew v1.1.1
-	github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea
+	github.com/deckarep/golang-set v1.8.0
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
 	github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf
 	github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48
@@ -32,7 +32,7 @@ require (
 	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa
 	github.com/google/uuid v1.1.5
 	github.com/gorilla/websocket v1.4.2
-	github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29
+	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/hashicorp/go-bexpr v0.1.10
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/holiman/bloomfilter/v2 v2.0.3
@@ -41,7 +41,7 @@ require (
 	github.com/influxdata/influxdb v1.8.3
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
-	github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
+	github.com/jackpal/go-nat-pmp v1.0.2
 	github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559

go.sum

@@ -111,8 +111,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0=
+github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4=
-github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo=
 github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
 github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU=
 github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw=
@@ -216,8 +216,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M=
+github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0=
-github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
 github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -248,8 +248,8 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y
 github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
-github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
+github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
-github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+Uj2hWd8aOlwPmoZ+CITRFrdit+sDGfAg8U=
 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=

graphql/graphql.go

@@ -372,6 +372,9 @@ func (t *Transaction) Status(ctx context.Context) (*Long, error) {
 	if err != nil || receipt == nil {
 		return nil, err
 	}
+	if len(receipt.PostState) != 0 {
+		return nil, nil
+	}
 	ret := Long(receipt.Status)
 	return &ret, nil
 }
@@ -596,21 +599,18 @@ func (b *Block) BaseFeePerGas(ctx context.Context) (*hexutil.Big, error) {
 }
 
 func (b *Block) Parent(ctx context.Context) (*Block, error) {
-	// If the block header hasn't been fetched, and we'll need it, fetch it.
+	if _, err := b.resolveHeader(ctx); err != nil {
-	if b.numberOrHash == nil && b.header == nil {
+		return nil, err
-		if _, err := b.resolveHeader(ctx); err != nil {
-			return nil, err
-		}
 	}
-	if b.header != nil && b.header.Number.Uint64() > 0 {
+	if b.header == nil || b.header.Number.Uint64() < 1 {
-		num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1))
+		return nil, nil
-		return &Block{
-			backend:      b.backend,
-			numberOrHash: &num,
-			hash:         b.header.ParentHash,
-		}, nil
 	}
-	return nil, nil
+	num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1))
+	return &Block{
+		backend:      b.backend,
+		numberOrHash: &num,
+		hash:         b.header.ParentHash,
+	}, nil
 }
 
 func (b *Block) Difficulty(ctx context.Context) (hexutil.Big, error) {
@@ -1110,10 +1110,21 @@ func (r *Resolver) Blocks(ctx context.Context, args struct {
 	ret := make([]*Block, 0, to-from+1)
 	for i := from; i <= to; i++ {
 		numberOrHash := rpc.BlockNumberOrHashWithNumber(i)
-		ret = append(ret, &Block{
+		block := &Block{
 			backend:      r.backend,
 			numberOrHash: &numberOrHash,
-		})
+		}
+		// Resolve the header to check for existence.
+		// Note we don't resolve block directly here since it will require an
+		// additional network request for light client.
+		h, err := block.resolveHeader(ctx)
+		if err != nil {
+			return nil, err
+		} else if h == nil {
+			// Blocks after must be non-existent too, break.
+			break
+		}
+		ret = append(ret, block)
 	}
 	return ret, nil
 }

graphql/graphql_test.go

@@ -48,6 +48,7 @@ func TestBuildSchema(t *testing.T) {
 	conf := node.DefaultConfig
 	conf.DataDir = ddir
 	stack, err := node.New(&conf)
+	defer stack.Close()
 	if err != nil {
 		t.Fatalf("could not create new node: %v", err)
 	}

interfaces.go

@@ -102,7 +102,12 @@ type SyncProgress struct {
 	CurrentBlock  uint64 // Current block number where sync is at
 	HighestBlock  uint64 // Highest alleged block number in the chain
 
-	// Fields belonging to snap sync
+	// "fast sync" fields. These used to be sent by geth, but are no longer used
+	// since version v1.10.
+	PulledStates uint64 // Number of state trie entries already downloaded
+	KnownStates  uint64 // Total number of state trie entries known about
+
+	// "snap sync" fields.
 	SyncedAccounts      uint64 // Number of accounts downloaded
 	SyncedAccountBytes  uint64 // Number of account trie bytes persisted to disk
 	SyncedBytecodes     uint64 // Number of bytecodes downloaded

internal/ethapi/api.go

@@ -767,8 +767,7 @@ func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Ha
 	return nil, err
 }
 
-// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true
+// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index.
-// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
 func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) {
 	block, err := s.b.BlockByNumber(ctx, blockNr)
 	if block != nil {
@@ -783,8 +782,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context,
 	return nil, err
 }
 
-// GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true
+// GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index.
-// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
 func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) {
 	block, err := s.b.BlockByHash(ctx, blockHash)
 	if block != nil {
@@ -1432,8 +1430,9 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH
 	} else {
 		to = crypto.CreateAddress(args.from(), uint64(*args.Nonce))
 	}
+	isPostMerge := header.Difficulty.Cmp(common.Big0) == 0
 	// Retrieve the precompiles since they don't need to be added to the access list
-	precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number))
+	precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number, isPostMerge))
 
 	// Create an initial tracer
 	prevTracer := logger.NewAccessListTracer(nil, args.from(), to, precompiles)

internal/ethapi/transaction_args.go

@@ -55,20 +55,20 @@ type TransactionArgs struct {
 }
 
 // from retrieves the transaction sender address.
-func (arg *TransactionArgs) from() common.Address {
+func (args *TransactionArgs) from() common.Address {
-	if arg.From == nil {
+	if args.From == nil {
 		return common.Address{}
 	}
-	return *arg.From
+	return *args.From
 }
 
 // data retrieves the transaction calldata. Input field is preferred.
-func (arg *TransactionArgs) data() []byte {
+func (args *TransactionArgs) data() []byte {
-	if arg.Input != nil {
+	if args.Input != nil {
-		return *arg.Input
+		return *args.Input
 	}
-	if arg.Data != nil {
+	if args.Data != nil {
-		return *arg.Data
+		return *args.Data
 	}
 	return nil
 }

internal/jsre/jsre_test.go

@@ -83,20 +83,20 @@ func TestNatto(t *testing.T) {
 
 	err := jsre.Exec("test.js")
 	if err != nil {
-		t.Errorf("expected no error, got %v", err)
+		t.Fatalf("expected no error, got %v", err)
 	}
 	time.Sleep(100 * time.Millisecond)
 	val, err := jsre.Run("msg")
 	if err != nil {
-		t.Errorf("expected no error, got %v", err)
+		t.Fatalf("expected no error, got %v", err)
 	}
 	if val.ExportType().Kind() != reflect.String {
-		t.Errorf("expected string value, got %v", val)
+		t.Fatalf("expected string value, got %v", val)
 	}
 	exp := "testMsg"
 	got := val.ToString().String()
 	if exp != got {
-		t.Errorf("expected '%v', got '%v'", exp, got)
+		t.Fatalf("expected '%v', got '%v'", exp, got)
 	}
 	jsre.Stop(false)
 }

internal/shutdowncheck/shutdown_tracker.go

@@ -0,0 +1,85 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package shutdowncheck
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ShutdownTracker is a service that reports previous unclean shutdowns
+// upon start. It needs to be started after a successful start-up and stopped
+// after a successful shutdown, just before the db is closed.
+type ShutdownTracker struct {
+	db     ethdb.Database
+	stopCh chan struct{}
+}
+
+// NewShutdownTracker creates a new ShutdownTracker instance and has
+// no other side-effect.
+func NewShutdownTracker(db ethdb.Database) *ShutdownTracker {
+	return &ShutdownTracker{
+		db:     db,
+		stopCh: make(chan struct{}),
+	}
+}
+
+// MarkStartup is to be called in the beginning when the node starts. It will:
+// - Push a new startup marker to the db
+// - Report previous unclean shutdowns
+func (t *ShutdownTracker) MarkStartup() {
+	if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(t.db); err != nil {
+		log.Error("Could not update unclean-shutdown-marker list", "error", err)
+	} else {
+		if discards > 0 {
+			log.Warn("Old unclean shutdowns found", "count", discards)
+		}
+		for _, tstamp := range uncleanShutdowns {
+			t := time.Unix(int64(tstamp), 0)
+			log.Warn("Unclean shutdown detected", "booted", t,
+				"age", common.PrettyAge(t))
+		}
+	}
+}
+
+// Start runs an event loop that updates the current marker's timestamp every 5 minutes.
+func (t *ShutdownTracker) Start() {
+	go func() {
+		ticker := time.NewTicker(5 * time.Minute)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				rawdb.UpdateUncleanShutdownMarker(t.db)
+			case <-t.stopCh:
+				return
+			}
+		}
+	}()
+}
+
+// Stop will stop the update loop and clear the current marker.
+func (t *ShutdownTracker) Stop() {
+	// Stop update loop.
+	t.stopCh <- struct{}{}
+	// Clear last marker.
+	rawdb.PopUncleanShutdownMarker(t.db)
+}

internal/web3ext/web3ext.go

@@ -576,6 +576,11 @@ web3._extend({
 			params: 3,
 			inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter, null]
 		}),
+		new web3._extend.Method({
+			name: 'getLogs',
+			call: 'eth_getLogs',
+			params: 1,
+		}),
 	],
 	properties: [
 		new web3._extend.Property({

les/catalyst/api.go

@@ -0,0 +1,178 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package catalyst implements the temporary eth1/eth2 RPC integration.
+package catalyst
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/beacon"
+	"github.com/ethereum/go-ethereum/les"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// Register adds catalyst APIs to the light client.
+func Register(stack *node.Node, backend *les.LightEthereum) error {
+	log.Warn("Catalyst mode enabled", "protocol", "les")
+	stack.RegisterAPIs([]rpc.API{
+		{
+			Namespace: "engine",
+			Version:   "1.0",
+			Service:   NewConsensusAPI(backend),
+			Public:    true,
+		},
+	})
+	return nil
+}
+
+type ConsensusAPI struct {
+	les *les.LightEthereum
+}
+
+// NewConsensusAPI creates a new consensus api for the given backend.
+// The underlying blockchain needs to have a valid terminal total difficulty set.
+func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI {
+	if les.BlockChain().Config().TerminalTotalDifficulty == nil {
+		panic("Catalyst started without valid total difficulty")
+	}
+	return &ConsensusAPI{les: les}
+}
+
+// ForkchoiceUpdatedV1 has several responsibilities:
+// If the method is called with an empty head block:
+// 		we return success, which can be used to check if the catalyst mode is enabled
+// If the total difficulty was not reached:
+// 		we return INVALID
+// If the finalizedBlockHash is set:
+// 		we check if we have the finalizedBlockHash in our db, if not we start a sync
+// We try to set our blockchain to the headBlock
+// If there are payloadAttributes:
+//      we return an error since block creation is not supported in les mode
+func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) {
+	if heads.HeadBlockHash == (common.Hash{}) {
+		return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil
+	}
+	if err := api.checkTerminalTotalDifficulty(heads.HeadBlockHash); err != nil {
+		if header := api.les.BlockChain().GetHeaderByHash(heads.HeadBlockHash); header == nil {
+			// TODO (MariusVanDerWijden) trigger sync
+			return beacon.SYNCING, nil
+		}
+		return beacon.INVALID, err
+	}
+	// If the finalized block is set, check if it is in our blockchain
+	if heads.FinalizedBlockHash != (common.Hash{}) {
+		if header := api.les.BlockChain().GetHeaderByHash(heads.FinalizedBlockHash); header == nil {
+			// TODO (MariusVanDerWijden) trigger sync
+			return beacon.SYNCING, nil
+		}
+	}
+	// SetHead
+	if err := api.setHead(heads.HeadBlockHash); err != nil {
+		return beacon.INVALID, err
+	}
+	if payloadAttributes != nil {
+		return beacon.INVALID, errors.New("not supported")
+	}
+	return beacon.ForkChoiceResponse{Status: beacon.SUCCESS.Status, PayloadID: nil}, nil
+}
+
+// GetPayloadV1 returns a cached payload by id. It's not supported in les mode.
+func (api *ConsensusAPI) GetPayloadV1(payloadID beacon.PayloadID) (*beacon.ExecutableDataV1, error) {
+	return nil, &beacon.GenericServerError
+}
+
+// ExecutePayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
+func (api *ConsensusAPI) ExecutePayloadV1(params beacon.ExecutableDataV1) (beacon.ExecutePayloadResponse, error) {
+	block, err := beacon.ExecutableDataToBlock(params)
+	if err != nil {
+		return api.invalid(), err
+	}
+	if !api.les.BlockChain().HasHeader(block.ParentHash(), block.NumberU64()-1) {
+		/*
+			TODO (MariusVanDerWijden) reenable once sync is merged
+			if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), block.Header()); err != nil {
+				return SYNCING, err
+			}
+		*/
+		// TODO (MariusVanDerWijden) we should return nil here not empty hash
+		return beacon.ExecutePayloadResponse{Status: beacon.SYNCING.Status, LatestValidHash: common.Hash{}}, nil
+	}
+	parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash)
+	if parent == nil {
+		return api.invalid(), fmt.Errorf("could not find parent %x", params.ParentHash)
+	}
+	td := api.les.BlockChain().GetTd(parent.Hash(), block.NumberU64()-1)
+	ttd := api.les.BlockChain().Config().TerminalTotalDifficulty
+	if td.Cmp(ttd) < 0 {
+		return api.invalid(), fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd)
+	}
+	if err = api.les.BlockChain().InsertHeader(block.Header()); err != nil {
+		return api.invalid(), err
+	}
+	if merger := api.les.Merger(); !merger.TDDReached() {
+		merger.ReachTTD()
+	}
+	return beacon.ExecutePayloadResponse{Status: beacon.VALID.Status, LatestValidHash: block.Hash()}, nil
+}
+
+// invalid returns a response "INVALID" with the latest valid hash set to the current head.
+func (api *ConsensusAPI) invalid() beacon.ExecutePayloadResponse {
+	return beacon.ExecutePayloadResponse{Status: beacon.INVALID.Status, LatestValidHash: api.les.BlockChain().CurrentHeader().Hash()}
+}
+
+func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error {
+	// shortcut if we entered PoS already
+	if api.les.Merger().PoSFinalized() {
+		return nil
+	}
+	// make sure the parent has enough terminal total difficulty
+	header := api.les.BlockChain().GetHeaderByHash(head)
+	if header == nil {
+		return &beacon.GenericServerError
+	}
+	td := api.les.BlockChain().GetTd(header.Hash(), header.Number.Uint64())
+	if td != nil && td.Cmp(api.les.BlockChain().Config().TerminalTotalDifficulty) < 0 {
+		return &beacon.InvalidTB
+	}
+	return nil
+}
+
+// setHead is called to perform a force choice.
+func (api *ConsensusAPI) setHead(newHead common.Hash) error {
+	log.Info("Setting head", "head", newHead)
+
+	headHeader := api.les.BlockChain().CurrentHeader()
+	if headHeader.Hash() == newHead {
+		return nil
+	}
+	newHeadHeader := api.les.BlockChain().GetHeaderByHash(newHead)
+	if newHeadHeader == nil {
+		return &beacon.GenericServerError
+	}
+	if err := api.les.BlockChain().SetChainHead(newHeadHeader); err != nil {
+		return err
+	}
+	// Trigger the transition if it's the first `NewHead` event.
+	if merger := api.les.Merger(); !merger.PoSFinalized() {
+		merger.FinalizePoS()
+	}
+	return nil
+}

les/catalyst/api_test.go

@@ -0,0 +1,244 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package catalyst
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/beacon"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/downloader"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
+	"github.com/ethereum/go-ethereum/les"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+var (
+	// testKey is a private key to use for funding a tester account.
+	testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+
+	// testAddr is the Ethereum address of the tester account.
+	testAddr = crypto.PubkeyToAddress(testKey.PublicKey)
+
+	testBalance = big.NewInt(2e18)
+)
+
+func generatePreMergeChain(n int) (*core.Genesis, []*types.Header, []*types.Block) {
+	db := rawdb.NewMemoryDatabase()
+	config := params.AllEthashProtocolChanges
+	genesis := &core.Genesis{
+		Config:    config,
+		Alloc:     core.GenesisAlloc{testAddr: {Balance: testBalance}},
+		ExtraData: []byte("test genesis"),
+		Timestamp: 9000,
+		BaseFee:   big.NewInt(params.InitialBaseFee),
+	}
+	gblock := genesis.ToBlock(db)
+	engine := ethash.NewFaker()
+	blocks, _ := core.GenerateChain(config, gblock, engine, db, n, nil)
+	totalDifficulty := big.NewInt(0)
+
+	var headers []*types.Header
+	for _, b := range blocks {
+		totalDifficulty.Add(totalDifficulty, b.Difficulty())
+		headers = append(headers, b.Header())
+	}
+	config.TerminalTotalDifficulty = totalDifficulty
+
+	return genesis, headers, blocks
+}
+
+func TestSetHeadBeforeTotalDifficulty(t *testing.T) {
+	genesis, headers, blocks := generatePreMergeChain(10)
+	n, lesService := startLesService(t, genesis, headers)
+	defer n.Close()
+
+	api := NewConsensusAPI(lesService)
+	fcState := beacon.ForkchoiceStateV1{
+		HeadBlockHash:      blocks[5].Hash(),
+		SafeBlockHash:      common.Hash{},
+		FinalizedBlockHash: common.Hash{},
+	}
+	if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err == nil {
+		t.Errorf("fork choice updated before total terminal difficulty should fail")
+	}
+}
+
+func TestExecutePayloadV1(t *testing.T) {
+	genesis, headers, blocks := generatePreMergeChain(10)
+	n, lesService := startLesService(t, genesis, headers[:9])
+	lesService.Merger().ReachTTD()
+	defer n.Close()
+
+	api := NewConsensusAPI(lesService)
+	fcState := beacon.ForkchoiceStateV1{
+		HeadBlockHash:      blocks[8].Hash(),
+		SafeBlockHash:      common.Hash{},
+		FinalizedBlockHash: common.Hash{},
+	}
+	if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil {
+		t.Errorf("Failed to update head %v", err)
+	}
+	block := blocks[9]
+
+	fakeBlock := types.NewBlock(&types.Header{
+		ParentHash:  block.ParentHash(),
+		UncleHash:   crypto.Keccak256Hash(nil),
+		Coinbase:    block.Coinbase(),
+		Root:        block.Root(),
+		TxHash:      crypto.Keccak256Hash(nil),
+		ReceiptHash: crypto.Keccak256Hash(nil),
+		Bloom:       block.Bloom(),
+		Difficulty:  big.NewInt(0),
+		Number:      block.Number(),
+		GasLimit:    block.GasLimit(),
+		GasUsed:     block.GasUsed(),
+		Time:        block.Time(),
+		Extra:       block.Extra(),
+		MixDigest:   block.MixDigest(),
+		Nonce:       types.BlockNonce{},
+		BaseFee:     block.BaseFee(),
+	}, nil, nil, nil, trie.NewStackTrie(nil))
+
+	_, err := api.ExecutePayloadV1(beacon.ExecutableDataV1{
+		ParentHash:    fakeBlock.ParentHash(),
+		FeeRecipient:  fakeBlock.Coinbase(),
+		StateRoot:     fakeBlock.Root(),
+		ReceiptsRoot:  fakeBlock.ReceiptHash(),
+		LogsBloom:     fakeBlock.Bloom().Bytes(),
+		Random:        fakeBlock.MixDigest(),
+		Number:        fakeBlock.NumberU64(),
+		GasLimit:      fakeBlock.GasLimit(),
+		GasUsed:       fakeBlock.GasUsed(),
+		Timestamp:     fakeBlock.Time(),
+		ExtraData:     fakeBlock.Extra(),
+		BaseFeePerGas: fakeBlock.BaseFee(),
+		BlockHash:     fakeBlock.Hash(),
+		Transactions:  encodeTransactions(fakeBlock.Transactions()),
+	})
+	if err != nil {
+		t.Errorf("Failed to execute payload %v", err)
+	}
+	headHeader := api.les.BlockChain().CurrentHeader()
+	if headHeader.Number.Uint64() != fakeBlock.NumberU64()-1 {
+		t.Fatal("Unexpected chain head update")
+	}
+	fcState = beacon.ForkchoiceStateV1{
+		HeadBlockHash:      fakeBlock.Hash(),
+		SafeBlockHash:      common.Hash{},
+		FinalizedBlockHash: common.Hash{},
+	}
+	if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil {
+		t.Fatal("Failed to update head")
+	}
+	headHeader = api.les.BlockChain().CurrentHeader()
+	if headHeader.Number.Uint64() != fakeBlock.NumberU64() {
+		t.Fatal("Failed to update chain head")
+	}
+}
+
+func TestEth2DeepReorg(t *testing.T) {
+	// TODO (MariusVanDerWijden) TestEth2DeepReorg is currently broken, because it tries to reorg
+	// before the totalTerminalDifficulty threshold
+	/*
+		genesis, preMergeBlocks := generatePreMergeChain(core.TriesInMemory * 2)
+		n, ethservice := startEthService(t, genesis, preMergeBlocks)
+		defer n.Close()
+
+		var (
+			api    = NewConsensusAPI(ethservice, nil)
+			parent = preMergeBlocks[len(preMergeBlocks)-core.TriesInMemory-1]
+			head   = ethservice.BlockChain().CurrentBlock().NumberU64()
+		)
+		if ethservice.BlockChain().HasBlockAndState(parent.Hash(), parent.NumberU64()) {
+			t.Errorf("Block %d not pruned", parent.NumberU64())
+		}
+		for i := 0; i < 10; i++ {
+			execData, err := api.assembleBlock(AssembleBlockParams{
+				ParentHash: parent.Hash(),
+				Timestamp:  parent.Time() + 5,
+			})
+			if err != nil {
+				t.Fatalf("Failed to create the executable data %v", err)
+			}
+			block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData)
+			if err != nil {
+				t.Fatalf("Failed to convert executable data to block %v", err)
+			}
+			newResp, err := api.ExecutePayload(*execData)
+			if err != nil || newResp.Status != "VALID" {
+				t.Fatalf("Failed to insert block: %v", err)
+			}
+			if ethservice.BlockChain().CurrentBlock().NumberU64() != head {
+				t.Fatalf("Chain head shouldn't be updated")
+			}
+			if err := api.setHead(block.Hash()); err != nil {
+				t.Fatalf("Failed to set head: %v", err)
+			}
+			if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() {
+				t.Fatalf("Chain head should be updated")
+			}
+			parent, head = block, block.NumberU64()
+		}
+	*/
+}
+
+// startEthService creates a full node instance for testing.
+func startLesService(t *testing.T, genesis *core.Genesis, headers []*types.Header) (*node.Node, *les.LightEthereum) {
+	t.Helper()
+
+	n, err := node.New(&node.Config{})
+	if err != nil {
+		t.Fatal("can't create node:", err)
+	}
+	ethcfg := &ethconfig.Config{
+		Genesis:        genesis,
+		Ethash:         ethash.Config{PowMode: ethash.ModeFake},
+		SyncMode:       downloader.LightSync,
+		TrieDirtyCache: 256,
+		TrieCleanCache: 256,
+		LightPeers:     10,
+	}
+	lesService, err := les.New(n, ethcfg)
+	if err != nil {
+		t.Fatal("can't create eth service:", err)
+	}
+	if err := n.Start(); err != nil {
+		t.Fatal("can't start node:", err)
+	}
+	if _, err := lesService.BlockChain().InsertHeaderChain(headers, 0); err != nil {
+		n.Close()
+		t.Fatal("can't import test headers:", err)
+	}
+	return n, lesService
+}
+
+func encodeTransactions(txs []*types.Transaction) [][]byte {
+	var enc = make([][]byte, len(txs))
+	for i, tx := range txs {
+		enc[i], _ = tx.MarshalBinary()
+	}
+	return enc
+}

les/client.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/internal/shutdowncheck"
 	"github.com/ethereum/go-ethereum/les/downloader"
 	"github.com/ethereum/go-ethereum/les/vflux"
 	vfc "github.com/ethereum/go-ethereum/les/vflux/client"
@@ -77,6 +78,8 @@ type LightEthereum struct {
 	p2pServer  *p2p.Server
 	p2pConfig  *p2p.Config
 	udpEnabled bool
+
+	shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
 }
 
 // New creates an instance of the light client.
@@ -107,17 +110,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
 			lesDb:       lesDb,
 			closeCh:     make(chan struct{}),
 		},
-		peers:          peers,
+		peers:           peers,
-		eventMux:       stack.EventMux(),
+		eventMux:        stack.EventMux(),
-		reqDist:        newRequestDistributor(peers, &mclock.System{}),
+		reqDist:         newRequestDistributor(peers, &mclock.System{}),
-		accountManager: stack.AccountManager(),
+		accountManager:  stack.AccountManager(),
-		merger:         merger,
+		merger:          merger,
-		engine:         ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb),
+		engine:          ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb),
-		bloomRequests:  make(chan chan *bloombits.Retrieval),
+		bloomRequests:   make(chan chan *bloombits.Retrieval),
-		bloomIndexer:   core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations),
+		bloomIndexer:    core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations),
-		p2pServer:      stack.Server(),
+		p2pServer:       stack.Server(),
-		p2pConfig:      &stack.Config().P2P,
+		p2pConfig:       &stack.Config().P2P,
-		udpEnabled:     stack.Config().P2P.DiscoveryV5,
+		udpEnabled:      stack.Config().P2P.DiscoveryV5,
+		shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb),
 	}
 
 	var prenegQuery vfc.QueryFunc
@@ -185,19 +189,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) {
 	stack.RegisterProtocols(leth.Protocols())
 	stack.RegisterLifecycle(leth)
 
-	// Check for unclean shutdown
+	// Successful startup; push a marker and check previous unclean shutdowns.
-	if uncleanShutdowns, discards, err := rawdb.PushUncleanShutdownMarker(chainDb); err != nil {
+	leth.shutdownTracker.MarkStartup()
-		log.Error("Could not update unclean-shutdown-marker list", "error", err)
+
-	} else {
-		if discards > 0 {
-			log.Warn("Old unclean shutdowns found", "count", discards)
-		}
-		for _, tstamp := range uncleanShutdowns {
-			t := time.Unix(int64(tstamp), 0)
-			log.Warn("Unclean shutdown detected", "booted", t,
-				"age", common.PrettyAge(t))
-		}
-	}
 	return leth, nil
 }
 
@@ -352,6 +346,9 @@ func (s *LightEthereum) Protocols() []p2p.Protocol {
 func (s *LightEthereum) Start() error {
 	log.Warn("Light client mode is an experimental feature")
 
+	// Regularly update shutdown marker
+	s.shutdownTracker.Start()
+
 	if s.udpEnabled && s.p2pServer.DiscV5 == nil {
 		s.udpEnabled = false
 		log.Error("Discovery v5 is not initialized")
@@ -387,7 +384,9 @@ func (s *LightEthereum) Stop() error {
 	s.engine.Close()
 	s.pruner.close()
 	s.eventMux.Stop()
-	rawdb.PopUncleanShutdownMarker(s.chainDb)
+	// Clean shutdown marker as the last thing before closing db
+	s.shutdownTracker.Stop()
+
 	s.chainDb.Close()
 	s.lesDb.Close()
 	s.wg.Wait()

les/server_handler.go

@@ -421,7 +421,10 @@ func (h *serverHandler) broadcastLoop() {
 			}
 			var reorg uint64
 			if lastHead != nil {
-				reorg = lastHead.Number.Uint64() - rawdb.FindCommonAncestor(h.chainDb, header, lastHead).Number.Uint64()
+				// If a setHead has been performed, the common ancestor can be nil.
+				if ancestor := rawdb.FindCommonAncestor(h.chainDb, header, lastHead); ancestor != nil {
+					reorg = lastHead.Number.Uint64() - ancestor.Number.Uint64()
+				}
 			}
 			lastHead, lastTd = header, td
 			log.Debug("Announcing block to peers", "number", number, "hash", hash, "td", td, "reorg", reorg)

miner/miner.go

@@ -35,10 +35,12 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
-// Backend wraps all methods required for mining.
+// Backend wraps all methods required for mining. Only full node is capable
+// to offer all the functions here.
 type Backend interface {
 	BlockChain() *core.BlockChain
 	TxPool() *core.TxPool
+	StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error)
 }
 
 // Config is the configuration parameters of mining.
@@ -68,7 +70,7 @@ type Miner struct {
 	wg sync.WaitGroup
 }
 
-func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool, merger *consensus.Merger) *Miner {
+func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(header *types.Header) bool) *Miner {
 	miner := &Miner{
 		eth:     eth,
 		mux:     mux,
@@ -76,7 +78,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even
 		exitCh:  make(chan struct{}),
 		startCh: make(chan common.Address),
 		stopCh:  make(chan struct{}),
-		worker:  newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true, merger),
+		worker:  newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true),
 	}
 	miner.wg.Add(1)
 	go miner.update()
@@ -233,6 +235,12 @@ func (miner *Miner) DisablePreseal() {
 	miner.worker.disablePreseal()
 }
 
+// GetSealingBlock retrieves a sealing block based on the given parameters.
+// The returned block is not sealed but all other fields should be filled.
+func (miner *Miner) GetSealingBlock(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash) (*types.Block, error) {
+	return miner.worker.getSealingBlock(parent, timestamp, coinbase, random)
+}
+
 // SubscribePendingLogs starts delivering logs from pending transactions
 // to the given channel.
 func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription {

miner/miner_test.go

@@ -18,11 +18,11 @@
 package miner
 
 import (
+	"errors"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -55,6 +55,10 @@ func (m *mockBackend) TxPool() *core.TxPool {
 	return m.txPool
 }
 
+func (m *mockBackend) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) {
+	return nil, errors.New("not supported")
+}
+
 type testBlockChain struct {
 	statedb       *state.StateDB
 	gasLimit      uint64
@@ -80,7 +84,8 @@ func (bc *testBlockChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent)
 }
 
 func TestMiner(t *testing.T) {
-	miner, mux := createMiner(t)
+	miner, mux, cleanup := createMiner(t)
+	defer cleanup(false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
 	// Start the downloader
@@ -107,7 +112,8 @@ func TestMiner(t *testing.T) {
 // An initial FailedEvent should allow mining to stop on a subsequent
 // downloader StartEvent.
 func TestMinerDownloaderFirstFails(t *testing.T) {
-	miner, mux := createMiner(t)
+	miner, mux, cleanup := createMiner(t)
+	defer cleanup(false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
 	// Start the downloader
@@ -138,8 +144,8 @@ func TestMinerDownloaderFirstFails(t *testing.T) {
 }
 
 func TestMinerStartStopAfterDownloaderEvents(t *testing.T) {
-	miner, mux := createMiner(t)
+	miner, mux, cleanup := createMiner(t)
-
+	defer cleanup(false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
 	// Start the downloader
@@ -161,7 +167,8 @@ func TestMinerStartStopAfterDownloaderEvents(t *testing.T) {
 }
 
 func TestStartWhileDownload(t *testing.T) {
-	miner, mux := createMiner(t)
+	miner, mux, cleanup := createMiner(t)
+	defer cleanup(false)
 	waitForMiningState(t, miner, false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
@@ -174,16 +181,19 @@ func TestStartWhileDownload(t *testing.T) {
 }
 
 func TestStartStopMiner(t *testing.T) {
-	miner, _ := createMiner(t)
+	miner, _, cleanup := createMiner(t)
+	defer cleanup(false)
 	waitForMiningState(t, miner, false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
 	miner.Stop()
 	waitForMiningState(t, miner, false)
+
 }
 
 func TestCloseMiner(t *testing.T) {
-	miner, _ := createMiner(t)
+	miner, _, cleanup := createMiner(t)
+	defer cleanup(true)
 	waitForMiningState(t, miner, false)
 	miner.Start(common.HexToAddress("0x12345"))
 	waitForMiningState(t, miner, true)
@@ -195,7 +205,8 @@ func TestCloseMiner(t *testing.T) {
 // TestMinerSetEtherbase checks that etherbase becomes set even if mining isn't
 // possible at the moment
 func TestMinerSetEtherbase(t *testing.T) {
-	miner, mux := createMiner(t)
+	miner, mux, cleanup := createMiner(t)
+	defer cleanup(false)
 	// Start with a 'bad' mining address
 	miner.Start(common.HexToAddress("0xdead"))
 	waitForMiningState(t, miner, true)
@@ -230,7 +241,7 @@ func waitForMiningState(t *testing.T, m *Miner, mining bool) {
 	t.Fatalf("Mining() == %t, want %t", state, mining)
 }
 
-func createMiner(t *testing.T) (*Miner, *event.TypeMux) {
+func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) {
 	// Create Ethash config
 	config := Config{
 		Etherbase: common.HexToAddress("123456789"),
@@ -246,7 +257,6 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) {
 	// Create consensus engine
 	engine := clique.New(chainConfig.Clique, chainDB)
 	// Create Ethereum backend
-	merger := consensus.NewMerger(rawdb.NewMemoryDatabase())
 	bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("can't create new chain %v", err)
@@ -259,5 +269,14 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) {
 	// Create event Mux
 	mux := new(event.TypeMux)
 	// Create Miner
-	return New(backend, &config, chainConfig, mux, engine, nil, merger), mux
+	miner := New(backend, &config, chainConfig, mux, engine, nil)
+	cleanup := func(skipMiner bool) {
+		bc.Stop()
+		engine.Close()
+		pool.Stop()
+		if !skipMiner {
+			miner.Close()
+		}
+	}
+	return miner, mux, cleanup
 }