feat(clique): allow shadowforking a clique network (#828) (#995)

* feat(clique): allow shadowforking a clique network (#828)

* minor fixes

* fix `TestShadowFork`

---------

Co-authored-by: Ömer Faruk Irmak <omerfirmak@gmail.com>

cmd/geth/main.go

@@ -152,6 +152,7 @@ var (
 		utils.L1DeploymentBlockFlag,
 		utils.CircuitCapacityCheckEnabledFlag,
 		utils.RollupVerifyEnabledFlag,
+		utils.ShadowforkPeersFlag,
 	}, utils.NetworkFlags, utils.DatabaseFlags)
 
 	rpcFlags = []cli.Flag{

cmd/utils/flags.go

@@ -1011,6 +1011,12 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Name:  "rpc.getlogs.maxrange",
 		Usage: "Limit max fetched block range for `eth_getLogs` method",
 	}
+
+	// Shadowfork peers
+	ShadowforkPeersFlag = &cli.StringSliceFlag{
+		Name:  "net.shadowforkpeers",
+		Usage: "peer ids of shadow fork peers",
+	}
 )
 
 var (
@@ -1811,6 +1817,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setCircuitCapacityCheck(ctx, cfg)
 	setEnableRollupVerify(ctx, cfg)
 	setMaxBlockRange(ctx, cfg)
+	if ctx.IsSet(ShadowforkPeersFlag.Name) {
+		cfg.ShadowForkPeerIDs = ctx.StringSlice(ShadowforkPeersFlag.Name)
+		log.Info("Shadow fork peers", "ids", cfg.ShadowForkPeerIDs)
+	}
 
 	// Cap the cache allowance and tune the garbage collector
 	mem, err := gopsutil.VirtualMemory()

consensus/clique/clique.go

@@ -68,6 +68,8 @@ var (
 
 	diffInTurn = big.NewInt(2) // Block difficulty for in-turn signatures
 	diffNoTurn = big.NewInt(1) // Block difficulty for out-of-turn signatures
+
+	diffShadowFork = diffNoTurn
 )
 
 // Various error messages to mark blocks invalid. These should be private to
@@ -195,6 +197,7 @@ func New(config *params.CliqueConfig, db ethdb.Database) *Clique {
 	if conf.Epoch == 0 {
 		conf.Epoch = epochLength
 	}
+
 	// Allocate the snapshot caches and create the engine
 	recents := lru.NewCache[common.Hash, *Snapshot](inmemorySnapshots)
 	signatures := lru.NewCache[common.Hash, common.Address](inmemorySignatures)
@@ -291,7 +294,7 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H
 	}
 	// Ensure that the block's difficulty is meaningful (may not be correct at this point)
 	if number > 0 {
-		if header.Difficulty == nil || (header.Difficulty.Cmp(diffInTurn) != 0 && header.Difficulty.Cmp(diffNoTurn) != 0) {
+		if header.Difficulty == nil || (header.Difficulty.Cmp(diffInTurn) != 0 && header.Difficulty.Cmp(diffNoTurn) != 0 && header.Difficulty.Cmp(diffShadowFork) != 0) {
 			return errInvalidDifficulty
 		}
 	}
@@ -376,6 +379,14 @@ func (c *Clique) snapshot(chain consensus.ChainHeaderReader, number uint64, hash
 		snap    *Snapshot
 	)
 	for snap == nil {
+		if c.config.ShadowForkHeight > 0 && number == c.config.ShadowForkHeight {
+			c.signatures.Purge()
+			c.recents.Purge()
+			c.proposals = make(map[common.Address]bool)
+			snap = newSnapshot(c.config, c.signatures, number, hash, []common.Address{c.config.ShadowForkSigner})
+			break
+		}
+
 		// If an in-memory snapshot was found, use that
 		if s, ok := c.recents.Get(hash); ok {
 			snap = s
@@ -486,11 +497,8 @@ func (c *Clique) verifySeal(snap *Snapshot, header *types.Header, parents []*typ
 	}
 	// Ensure that the difficulty corresponds to the turn-ness of the signer
 	if !c.fakeDiff {
-		inturn := snap.inturn(header.Number.Uint64(), signer)
-		if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
-			return errWrongDifficulty
-		}
-		if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
+		expected := c.calcDifficulty(snap, signer)
+		if header.Difficulty.Cmp(expected) != 0 {
 			return errWrongDifficulty
 		}
 	}
@@ -535,7 +543,7 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header
 	c.lock.RUnlock()
 
 	// Set the correct difficulty
-	header.Difficulty = calcDifficulty(snap, signer)
+	header.Difficulty = c.calcDifficulty(snap, signer)
 
 	// Ensure the extra data has all its components
 	if len(header.Extra) < extraVanity {
@@ -683,10 +691,14 @@ func (c *Clique) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64,
 	c.lock.RLock()
 	signer := c.signer
 	c.lock.RUnlock()
-	return calcDifficulty(snap, signer)
+	return c.calcDifficulty(snap, signer)
 }
 
-func calcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
+func (c *Clique) calcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
+	if c.config.ShadowForkHeight > 0 && snap.Number >= c.config.ShadowForkHeight {
+		// if we are past shadow fork point, set a low difficulty so that mainnet nodes don't try to switch to forked chain
+		return new(big.Int).Set(diffShadowFork)
+	}
 	if snap.inturn(snap.Number+1, signer) {
 		return new(big.Int).Set(diffInTurn)
 	}

consensus/clique/clique_test.go

@@ -17,7 +17,9 @@
 package clique
 
 import (
+	"bytes"
 	"math/big"
+	"strings"
 	"testing"
 
 	"github.com/scroll-tech/go-ethereum/common"
@@ -27,6 +29,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/core/vm"
 	"github.com/scroll-tech/go-ethereum/crypto"
 	"github.com/scroll-tech/go-ethereum/params"
+	"github.com/scroll-tech/go-ethereum/trie"
 )
 
 // This test case is a repro of an annoying bug that took us forever to catch.
@@ -123,3 +126,91 @@ func TestSealHash(t *testing.T) {
 		t.Errorf("have %x, want %x", have, want)
 	}
 }
+
+func TestShadowFork(t *testing.T) {
+	engineConf := *params.AllCliqueProtocolChanges.Clique
+	engineConf.Epoch = 2
+	forkedEngineConf := engineConf
+	forkedEngineConf.ShadowForkHeight = 3
+	shadowForkKey, _ := crypto.HexToECDSA(strings.Repeat("11", 32))
+	shadowForkAddr := crypto.PubkeyToAddress(shadowForkKey.PublicKey)
+	forkedEngineConf.ShadowForkSigner = shadowForkAddr
+
+	// Initialize a Clique chain with a single signer
+	var (
+		db           = rawdb.NewMemoryDatabase()
+		key, _       = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		addr         = crypto.PubkeyToAddress(key.PublicKey)
+		engine       = New(&engineConf, db)
+		signer       = new(types.HomesteadSigner)
+		forkedEngine = New(&forkedEngineConf, db)
+	)
+	genspec := &core.Genesis{
+		Config:    params.AllCliqueProtocolChanges,
+		ExtraData: make([]byte, extraVanity+common.AddressLength+extraSeal),
+		Alloc: map[common.Address]core.GenesisAccount{
+			addr: {Balance: big.NewInt(10000000000000000)},
+		},
+		BaseFee: big.NewInt(params.InitialBaseFee),
+	}
+	copy(genspec.ExtraData[extraVanity:], addr[:])
+	genesis := genspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
+
+	// Generate a batch of blocks, each properly signed
+	chain, _ := core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil)
+	defer chain.Stop()
+
+	forkedChain, _ := core.NewBlockChain(db, nil, genspec, nil, forkedEngine, vm.Config{}, nil, nil)
+	defer forkedChain.Stop()
+
+	blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, forkedEngine, db, 16, func(i int, block *core.BlockGen) {
+		// The chain maker doesn't have access to a chain, so the difficulty will be
+		// lets unset (nil). Set it here to the correct value.
+		if block.Number().Uint64() > forkedEngineConf.ShadowForkHeight {
+			block.SetDifficulty(diffShadowFork)
+		} else {
+			block.SetDifficulty(diffInTurn)
+		}
+
+		tx, err := types.SignTx(types.NewTransaction(block.TxNonce(addr), common.Address{0x00}, new(big.Int), params.TxGas, block.BaseFee(), nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTxWithChain(chain, tx)
+	})
+	for i, block := range blocks {
+		header := block.Header()
+		if i > 0 {
+			header.ParentHash = blocks[i-1].Hash()
+		}
+
+		signingAddr, signingKey := addr, key
+		if header.Number.Uint64() > forkedEngineConf.ShadowForkHeight {
+			// start signing with shadow fork authority key
+			signingAddr, signingKey = shadowForkAddr, shadowForkKey
+		}
+
+		header.Extra = make([]byte, extraVanity)
+		if header.Number.Uint64()%engineConf.Epoch == 0 {
+			header.Extra = append(header.Extra, signingAddr.Bytes()...)
+		}
+		header.Extra = append(header.Extra, bytes.Repeat([]byte{0}, extraSeal)...)
+
+		sig, _ := crypto.Sign(SealHash(header).Bytes(), signingKey)
+		copy(header.Extra[len(header.Extra)-extraSeal:], sig)
+		blocks[i] = block.WithSeal(header)
+	}
+
+	if _, err := chain.InsertChain(blocks); err == nil {
+		t.Fatalf("should've failed to insert some blocks to canonical chain")
+	}
+	if chain.CurrentHeader().Number.Uint64() != forkedEngineConf.ShadowForkHeight {
+		t.Fatalf("unexpected canonical chain height")
+	}
+	if _, err := forkedChain.InsertChain(blocks); err != nil {
+		t.Fatalf("failed to insert blocks to forked chain: %v %d", err, forkedChain.CurrentHeader().Number)
+	}
+	if forkedChain.CurrentHeader().Number.Uint64() != uint64(len(blocks)) {
+		t.Fatalf("unexpected forked chain height")
+	}
+}

consensus/misc/eip1559/eip1559_scroll.go

@@ -52,6 +52,9 @@ func VerifyEIP1559Header(config *params.ChainConfig, parent, header *types.Heade
 
 // CalcBaseFee calculates the basefee of the header.
 func CalcBaseFee(config *params.ChainConfig, parent *types.Header, parentL1BaseFee *big.Int) *big.Int {
+	if config.Clique != nil && config.Clique.ShadowForkHeight != 0 && parent.Number.Uint64() >= config.Clique.ShadowForkHeight {
+		return big.NewInt(10000000) // 0.01 Gwei
+	}
 	l2SequencerFee := big.NewInt(1000000) // 0.001 Gwei
 	provingFee := big.NewInt(33700000)    // 0.0337 Gwei
 

eth/backend.go

@@ -267,6 +267,8 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client sync_service.EthCl
 		BloomCache:     uint64(cacheLimit),
 		EventMux:       eth.eventMux,
 		RequiredBlocks: config.RequiredBlocks,
+
+		ShadowForkPeerIDs: config.ShadowForkPeerIDs,
 	}); err != nil {
 		return nil, err
 	}

eth/ethconfig/config.go

@@ -178,6 +178,9 @@ type Config struct {
 
 	// Max block range for eth_getLogs api method
 	MaxBlockRange int64
+
+	// List of peer ids that take part in the shadow-fork
+	ShadowForkPeerIDs []string
 }
 
 // CreateConsensusEngine creates a consensus engine for the given chain config.

eth/handler.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"math"
 	"math/big"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -93,6 +94,8 @@ type handlerConfig struct {
 	BloomCache     uint64                 // Megabytes to alloc for snap sync bloom
 	EventMux       *event.TypeMux         // Legacy event mux, deprecate for `feed`
 	RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
+
+	ShadowForkPeerIDs []string // List of peer ids that take part in the shadow-fork
 }
 
 type handler struct {
@@ -128,6 +131,8 @@ type handler struct {
 
 	handlerStartCh chan struct{}
 	handlerDoneCh  chan struct{}
+
+	shadowForkPeerIDs []string
 }
 
 // newHandler returns a handler for all Ethereum chain management protocol.
@@ -149,6 +154,8 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		quitSync:       make(chan struct{}),
 		handlerDoneCh:  make(chan struct{}),
 		handlerStartCh: make(chan struct{}),
+
+		shadowForkPeerIDs: config.ShadowForkPeerIDs,
 	}
 	if config.Sync == downloader.FullSync {
 		// Currently in Scroll we only support full sync,
@@ -269,7 +276,13 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		}
 		return h.chain.InsertChain(blocks)
 	}
-	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, h.removePeer)
+
+	fetcherDropPeerFunc := h.removePeer
+	// If we are shadowforking, don't drop peers.
+	if config.ShadowForkPeerIDs != nil {
+		fetcherDropPeerFunc = func(id string) {}
+	}
+	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock, heighter, nil, inserter, fetcherDropPeerFunc)
 
 	fetchTx := func(peer string, hashes []common.Hash) error {
 		p := h.peers.peer(peer)
@@ -396,7 +409,9 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error {
 
 	// Propagate existing transactions. new transactions appearing
 	// after this will be sent via broadcasts.
-	h.syncTransactions(peer)
+	if h.shadowForkPeerIDs == nil || slices.Contains(h.shadowForkPeerIDs, peer.ID()) {
+		h.syncTransactions(peer)
+	}
 
 	// Create a notification channel for pending requests if the peer goes down
 	dead := make(chan struct{})
@@ -567,7 +582,7 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) {
 		}
 	}
 	hash := block.Hash()
-	peers := h.peers.peersWithoutBlock(hash)
+	peers := onlyShadowForkPeers(h.shadowForkPeerIDs, h.peers.peersWithoutBlock(hash))
 
 	// If propagation is requested, send to a subset of the peer
 	if propagate {
@@ -620,7 +635,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) {
 			continue
 		}
 
-		peers := h.peers.peersWithoutTransaction(tx.Hash())
+		peers := onlyShadowForkPeers(h.shadowForkPeerIDs, h.peers.peersWithoutTransaction(tx.Hash()))
 
 		var numDirect int
 		switch {
@@ -695,3 +710,16 @@ func (h *handler) enableSyncedFeatures() {
 		h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize)
 	}
 }
+
+// onlyShadowForkPeers filters out peers that are not part of the shadow fork
+func onlyShadowForkPeers[peerT interface {
+	ID() string
+}](shadowForkPeerIDs []string, peers []peerT) []peerT {
+	if shadowForkPeerIDs == nil {
+		return peers
+	}
+
+	return slices.DeleteFunc(peers, func(peer peerT) bool {
+		return !slices.Contains(shadowForkPeerIDs, peer.ID())
+	})
+}

eth/handler_test.go

@@ -20,6 +20,9 @@ import (
 	"math/big"
 	"sort"
 	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
 
 	"github.com/scroll-tech/go-ethereum/common"
 	"github.com/scroll-tech/go-ethereum/consensus"
@@ -183,3 +186,82 @@ func (b *testHandler) close() {
 	b.handler.Stop()
 	b.chain.Stop()
 }
+
+type testPeer struct {
+	id string
+}
+
+func (p testPeer) ID() string {
+	return p.id
+}
+
+func TestOnlyShadowForkPeers(t *testing.T) {
+
+	tests := map[string]struct {
+		shadowForkPeerIDs []string
+		peers             []testPeer
+		expectedPeerIDs   []string
+	}{
+		"nil peers": {
+			shadowForkPeerIDs: nil,
+			peers:             nil,
+			expectedPeerIDs:   []string{},
+		},
+		"empty peers": {
+			shadowForkPeerIDs: nil,
+			peers:             []testPeer{},
+			expectedPeerIDs:   []string{},
+		},
+		"no fork": {
+			shadowForkPeerIDs: nil,
+			peers: []testPeer{
+				{
+					id: "peer1",
+				},
+				{
+					id: "peer2",
+				},
+			},
+			expectedPeerIDs: []string{
+				"peer1",
+				"peer2",
+			},
+		},
+		"some shadow fork peers": {
+			shadowForkPeerIDs: []string{"peer2"},
+			peers: []testPeer{
+				{
+					id: "peer1",
+				},
+				{
+					id: "peer2",
+				},
+			},
+			expectedPeerIDs: []string{
+				"peer2",
+			},
+		},
+		"no shadow fork peers": {
+			shadowForkPeerIDs: []string{"peer2"},
+			peers: []testPeer{
+				{
+					id: "peer1",
+				},
+				{
+					id: "peer3",
+				},
+			},
+			expectedPeerIDs: []string{},
+		},
+	}
+
+	for desc, test := range tests {
+		t.Run(desc, func(t *testing.T) {
+			gotIds := []string{}
+			for _, peer := range onlyShadowForkPeers(test.shadowForkPeerIDs, test.peers) {
+				gotIds = append(gotIds, peer.ID())
+			}
+			require.Equal(t, gotIds, test.expectedPeerIDs)
+		})
+	}
+}

eth/peerset.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"slices"
 	"sync"
 
 	"github.com/scroll-tech/go-ethereum/common"
@@ -232,7 +233,7 @@ func (ps *peerSet) snapLen() int {
 
 // peerWithHighestTD retrieves the known peer with the currently highest total
 // difficulty, but below the given PoS switchover threshold.
-func (ps *peerSet) peerWithHighestTD() *eth.Peer {
+func (ps *peerSet) peerWithHighestTD(whileList []string) *eth.Peer {
 	ps.lock.RLock()
 	defer ps.lock.RUnlock()
 
@@ -241,6 +242,9 @@ func (ps *peerSet) peerWithHighestTD() *eth.Peer {
 		bestTd   *big.Int
 	)
 	for _, p := range ps.peers {
+		if whileList != nil && !slices.Contains(whileList, p.ID()) {
+			continue
+		}
 		if _, td := p.Head(); bestPeer == nil || td.Cmp(bestTd) > 0 {
 			bestPeer, bestTd = p.Peer, td
 		}

eth/sync.go

@@ -162,10 +162,20 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp {
 	if cs.handler.peers.len() < minPeers {
 		return nil
 	}
+
+	var syncWhiteList []string
+	chainConfig := cs.handler.chain.Config()
+	currentHeight := cs.handler.chain.CurrentHeader().Number.Uint64()
+	if chainConfig.Clique != nil {
+		shadowForkHeight := chainConfig.Clique.ShadowForkHeight
+		if shadowForkHeight != 0 && currentHeight >= shadowForkHeight {
+			syncWhiteList = cs.handler.shadowForkPeerIDs
+		}
+	}
 	// We have enough peers, pick the one with the highest TD, but avoid going
 	// over the terminal total difficulty. Above that we expect the consensus
 	// clients to direct the chain head to sync to.
-	peer := cs.handler.peers.peerWithHighestTD()
+	peer := cs.handler.peers.peerWithHighestTD(syncWhiteList)
 	if peer == nil {
 		return nil
 	}

eth/sync_test.go

@@ -88,7 +88,7 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
 	time.Sleep(250 * time.Millisecond)
 
 	// Check that snap sync was disabled
-	op := peerToSyncOp(downloader.SnapSync, empty.handler.peers.peerWithHighestTD())
+	op := peerToSyncOp(downloader.SnapSync, empty.handler.peers.peerWithHighestTD([]string{}))
 	if err := empty.handler.doSync(op); err != nil {
 		t.Fatal("sync failed:", err)
 	}

params/config.go

@@ -611,7 +611,9 @@ type CliqueConfig struct {
 	Period uint64 `json:"period"` // Number of seconds between blocks to enforce
 	Epoch  uint64 `json:"epoch"`  // Epoch length to reset votes and checkpoint
 
-	RelaxedPeriod bool `json:"relaxed_period"` // Relaxes the period to be just an upper bound
+	RelaxedPeriod    bool           `json:"relaxed_period"`     // Relaxes the period to be just an upper bound
+	ShadowForkHeight uint64         `json:"shadow_fork_height"` // Allows shadow forking consensus layer at given height
+	ShadowForkSigner common.Address `json:"shadow_fork_signer"` // Sets the address to be the authorized signer after the shadow fork
 }
 
 // String implements the stringer interface, returning the consensus engine details.