Merge branch 'ethereum:master' into portal

cmd/geth/chaincmd.go

@@ -100,7 +100,7 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.MetricsInfluxDBOrganizationFlag,
 			utils.TxLookupLimitFlag,
 			utils.VMTraceFlag,
-			utils.VMTraceConfigFlag,
+			utils.VMTraceJsonConfigFlag,
 			utils.TransactionHistoryFlag,
 			utils.StateHistoryFlag,
 		}, utils.DatabaseFlags),

cmd/geth/main.go

@@ -137,7 +137,7 @@ var (
 		utils.DeveloperPeriodFlag,
 		utils.VMEnableDebugFlag,
 		utils.VMTraceFlag,
-		utils.VMTraceConfigFlag,
+		utils.VMTraceJsonConfigFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
 		utils.NoCompactionFlag,

cmd/utils/flags.go

@@ -545,7 +545,7 @@ var (
 		Usage:    "Name of tracer which should record internal VM operations (costly)",
 		Category: flags.VMCategory,
 	}
-	VMTraceConfigFlag = &cli.StringFlag{
+	VMTraceJsonConfigFlag = &cli.StringFlag{
 		Name:     "vmtrace.jsonconfig",
 		Usage:    "Tracer configuration (JSON)",
 		Category: flags.VMCategory,
@@ -1971,12 +1971,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(VMTraceFlag.Name) {
 		if name := ctx.String(VMTraceFlag.Name); name != "" {
 			var config string
-			if ctx.IsSet(VMTraceConfigFlag.Name) {
+			if ctx.IsSet(VMTraceJsonConfigFlag.Name) {
-				config = ctx.String(VMTraceConfigFlag.Name)
+				config = ctx.String(VMTraceJsonConfigFlag.Name)
 			}
 
 			cfg.VMTrace = name
-			cfg.VMTraceConfig = config
+			cfg.VMTraceJsonConfig = config
 		}
 	}
 }
@@ -2260,8 +2260,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	if ctx.IsSet(VMTraceFlag.Name) {
 		if name := ctx.String(VMTraceFlag.Name); name != "" {
 			var config json.RawMessage
-			if ctx.IsSet(VMTraceConfigFlag.Name) {
+			if ctx.IsSet(VMTraceJsonConfigFlag.Name) {
-				config = json.RawMessage(ctx.String(VMTraceConfigFlag.Name))
+				config = json.RawMessage(ctx.String(VMTraceJsonConfigFlag.Name))
 			}
 			t, err := tracers.LiveDirectory.New(name, config)
 			if err != nil {

core/blockchain.go

@@ -1153,6 +1153,10 @@ func (bc *BlockChain) Stop() {
 			}
 		}
 	}
+	// Allow tracers to clean-up and release resources.
+	if bc.logger != nil && bc.logger.OnClose != nil {
+		bc.logger.OnClose()
+	}
 	// Close the trie database, release all the held resources as the last step.
 	if err := bc.triedb.Close(); err != nil {
 		log.Error("Failed to close trie database", "err", err)

core/tracing/hooks.go

@@ -107,6 +107,9 @@ type (
 	// BlockchainInitHook is called when the blockchain is initialized.
 	BlockchainInitHook = func(chainConfig *params.ChainConfig)
 
+	// CloseHook is called when the blockchain closes.
+	CloseHook = func()
+
 	// BlockStartHook is called before executing `block`.
 	// `td` is the total difficulty prior to `block`.
 	BlockStartHook = func(event BlockEvent)
@@ -153,6 +156,7 @@ type Hooks struct {
 	OnGasChange GasChangeHook
 	// Chain events
 	OnBlockchainInit BlockchainInitHook
+	OnClose          CloseHook
 	OnBlockStart     BlockStartHook
 	OnBlockEnd       BlockEndHook
 	OnSkippedBlock   SkippedBlockHook

eth/backend.go

@@ -203,8 +203,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	)
 	if config.VMTrace != "" {
 		var traceConfig json.RawMessage
-		if config.VMTraceConfig != "" {
+		if config.VMTraceJsonConfig != "" {
-			traceConfig = json.RawMessage(config.VMTraceConfig)
+			traceConfig = json.RawMessage(config.VMTraceJsonConfig)
 		}
 		t, err := tracers.LiveDirectory.New(config.VMTrace, traceConfig)
 		if err != nil {

eth/downloader/downloader_test.go

@@ -1311,3 +1311,84 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) {
 		})
 	}
 }
+
+// Tests that synchronisation progress (origin block number and highest block
+// number) is tracked and updated correctly in case of manual head reversion
+func TestBeaconForkedSyncProgress68Full(t *testing.T) {
+	testBeaconForkedSyncProgress(t, eth.ETH68, FullSync)
+}
+func TestBeaconForkedSyncProgress68Snap(t *testing.T) {
+	testBeaconForkedSyncProgress(t, eth.ETH68, SnapSync)
+}
+func TestBeaconForkedSyncProgress68Light(t *testing.T) {
+	testBeaconForkedSyncProgress(t, eth.ETH68, LightSync)
+}
+
+func testBeaconForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) {
+	success := make(chan struct{})
+	tester := newTesterWithNotification(t, func() {
+		success <- struct{}{}
+	})
+	defer tester.terminate()
+
+	chainA := testChainForkLightA.shorten(len(testChainBase.blocks) + MaxHeaderFetch)
+	chainB := testChainForkLightB.shorten(len(testChainBase.blocks) + MaxHeaderFetch)
+
+	// Set a sync init hook to catch progress changes
+	starting := make(chan struct{})
+	progress := make(chan struct{})
+
+	tester.downloader.syncInitHook = func(origin, latest uint64) {
+		starting <- struct{}{}
+		<-progress
+	}
+	checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{})
+
+	// Synchronise with one of the forks and check progress
+	tester.newPeer("fork A", protocol, chainA.blocks[1:])
+	pending := new(sync.WaitGroup)
+	pending.Add(1)
+	go func() {
+		defer pending.Done()
+		if err := tester.downloader.BeaconSync(mode, chainA.blocks[len(chainA.blocks)-1].Header(), nil); err != nil {
+			panic(fmt.Sprintf("failed to beacon sync: %v", err))
+		}
+	}()
+
+	<-starting
+	progress <- struct{}{}
+	select {
+	case <-success:
+		checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+			HighestBlock: uint64(len(chainA.blocks) - 1),
+			CurrentBlock: uint64(len(chainA.blocks) - 1),
+		})
+	case <-time.NewTimer(time.Second * 3).C:
+		t.Fatalf("Failed to sync chain in three seconds")
+	}
+
+	// Set the head to a second fork
+	tester.newPeer("fork B", protocol, chainB.blocks[1:])
+	pending.Add(1)
+	go func() {
+		defer pending.Done()
+		if err := tester.downloader.BeaconSync(mode, chainB.blocks[len(chainB.blocks)-1].Header(), nil); err != nil {
+			panic(fmt.Sprintf("failed to beacon sync: %v", err))
+		}
+	}()
+
+	<-starting
+	progress <- struct{}{}
+
+	// reorg below available state causes the state sync to rewind to genesis
+	select {
+	case <-success:
+		checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{
+			HighestBlock:  uint64(len(chainB.blocks) - 1),
+			CurrentBlock:  uint64(len(chainB.blocks) - 1),
+			StartingBlock: 0,
+		})
+	case <-time.NewTimer(time.Second * 3).C:
+		t.Fatalf("Failed to sync chain in three seconds")
+	}
+}

eth/downloader/skeleton.go

@@ -1132,6 +1132,16 @@ func (s *skeleton) cleanStales(filled *types.Header) error {
 	if number+1 == s.progress.Subchains[0].Tail {
 		return nil
 	}
+	// If the latest fill was on a different subchain, it means the backfiller
+	// was interrupted before it got to do any meaningful work, no cleanup
+	header := rawdb.ReadSkeletonHeader(s.db, filled.Number.Uint64())
+	if header == nil {
+		log.Debug("Filled header outside of skeleton range", "number", number, "head", s.progress.Subchains[0].Head, "tail", s.progress.Subchains[0].Tail)
+		return nil
+	} else if header.Hash() != filled.Hash() {
+		log.Debug("Filled header on different sidechain", "number", number, "filled", filled.Hash(), "skeleton", header.Hash())
+		return nil
+	}
 	var (
 		start uint64
 		end   uint64

eth/ethconfig/config.go

@@ -142,8 +142,8 @@ type Config struct {
 	EnablePreimageRecording bool
 
 	// Enables VM tracing
-	VMTrace       string
+	VMTrace           string
-	VMTraceConfig string
+	VMTraceJsonConfig string
 
 	// Miscellaneous options
 	DocRoot string `toml:"-"`

eth/ethconfig/gen_config.go

@@ -51,7 +51,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		GPO                     gasprice.Config
 		EnablePreimageRecording bool
 		VMTrace                 string
-		VMTraceConfig           string
+		VMTraceJsonConfig       string
 		DocRoot                 string `toml:"-"`
 		RPCGasCap               uint64
 		RPCEVMTimeout           time.Duration
@@ -94,7 +94,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.GPO = c.GPO
 	enc.EnablePreimageRecording = c.EnablePreimageRecording
 	enc.VMTrace = c.VMTrace
-	enc.VMTraceConfig = c.VMTraceConfig
+	enc.VMTraceJsonConfig = c.VMTraceJsonConfig
 	enc.DocRoot = c.DocRoot
 	enc.RPCGasCap = c.RPCGasCap
 	enc.RPCEVMTimeout = c.RPCEVMTimeout
@@ -141,7 +141,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		GPO                     *gasprice.Config
 		EnablePreimageRecording *bool
 		VMTrace                 *string
-		VMTraceConfig           *string
+		VMTraceJsonConfig       *string
 		DocRoot                 *string `toml:"-"`
 		RPCGasCap               *uint64
 		RPCEVMTimeout           *time.Duration
@@ -255,8 +255,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.VMTrace != nil {
 		c.VMTrace = *dec.VMTrace
 	}
-	if dec.VMTraceConfig != nil {
+	if dec.VMTraceJsonConfig != nil {
-		c.VMTraceConfig = *dec.VMTraceConfig
+		c.VMTraceJsonConfig = *dec.VMTraceJsonConfig
 	}
 	if dec.DocRoot != nil {
 		c.DocRoot = *dec.DocRoot

eth/state_accessor.go

@@ -233,6 +233,12 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block,
 	if err != nil {
 		return nil, vm.BlockContext{}, nil, nil, err
 	}
+	// Insert parent beacon block root in the state as per EIP-4788.
+	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
+		context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil)
+		vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, eth.blockchain.Config(), vm.Config{})
+		core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+	}
 	if txIndex == 0 && len(block.Transactions()) == 0 {
 		return nil, vm.BlockContext{}, statedb, release, nil
 	}

eth/tracers/api.go

@@ -376,6 +376,13 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed
 				failed = err
 				break
 			}
+			// Insert block's parent beacon block root in the state
+			// as per EIP-4788.
+			if beaconRoot := next.BeaconRoot(); beaconRoot != nil {
+				context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil)
+				vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{})
+				core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+			}
 			// Clean out any pending release functions of trace state. Note this
 			// step must be done after constructing tracing state, because the
 			// tracing state of block next depends on the parent state and construction
@@ -517,7 +524,6 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config
 		return nil, err
 	}
 	defer release()
-
 	var (
 		roots              []common.Hash
 		signer             = types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time())
@@ -525,6 +531,10 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config
 		vmctx              = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil)
 		deleteEmptyObjects = chainConfig.IsEIP158(block.Number())
 	)
+	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
+		vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{})
+		core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+	}
 	for i, tx := range block.Transactions() {
 		if err := ctx.Err(); err != nil {
 			return nil, err
@@ -584,7 +594,6 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac
 		return nil, err
 	}
 	defer release()
-
 	// JS tracers have high overhead. In this case run a parallel
 	// process that generates states in one thread and traces txes
 	// in separate worker threads.
@@ -601,6 +610,10 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac
 		signer    = types.MakeSigner(api.backend.ChainConfig(), block.Number(), block.Time())
 		results   = make([]*txTraceResult, len(txs))
 	)
+	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
+		vmenv := vm.NewEVM(blockCtx, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{})
+		core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+	}
 	for i, tx := range txs {
 		// Generate the next state snapshot fast without tracing
 		msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee())
@@ -727,7 +740,6 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block
 		return nil, err
 	}
 	defer release()
-
 	// Retrieve the tracing configurations, or use default values
 	var (
 		logConfig logger.Config
@@ -756,6 +768,10 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block
 		// Note: This copies the config, to not screw up the main config
 		chainConfig, canon = overrideConfig(chainConfig, config.Overrides)
 	}
+	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
+		vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{})
+		core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+	}
 	for i, tx := range block.Transactions() {
 		// Prepare the transaction for un-traced execution
 		var (