diff --git a/accounts/abi/topics.go b/accounts/abi/topics.go index 7ce9b7273c..4819334ae6 100644 --- a/accounts/abi/topics.go +++ b/accounts/abi/topics.go @@ -42,7 +42,7 @@ func MakeTopics(query ...[]interface{}) ([][]common.Hash, error) { case common.Address: copy(topic[common.HashLength-common.AddressLength:], rule[:]) case *big.Int: - copy(topic[:], math.U256Bytes(rule)) + copy(topic[:], math.U256Bytes(new(big.Int).Set(rule))) case bool: if rule { topic[common.HashLength-1] = 1 diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index 6a4c50078a..161867e2d9 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -149,6 +149,23 @@ func TestMakeTopics(t *testing.T) { } }) } + + t.Run("does not mutate big.Int", func(t *testing.T) { + t.Parallel() + want := [][]common.Hash{{common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")}} + + in := big.NewInt(-1) + got, err := MakeTopics([]interface{}{in}) + if err != nil { + t.Fatalf("makeTopics() error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("makeTopics() = %v, want %v", got, want) + } + if orig := big.NewInt(-1); in.Cmp(orig) != 0 { + t.Fatalf("makeTopics() mutated an input parameter from %v to %v", orig, in) + } + }) } type args struct { diff --git a/build/ci.go b/build/ci.go index 754d88a86a..e3115af46d 100644 --- a/build/ci.go +++ b/build/ci.go @@ -256,7 +256,7 @@ func buildFlags(env build.Environment, staticLinking bool, buildTags []string) ( // See https://sourceware.org/binutils/docs-2.23.1/ld/Options.html#Options // regarding the options --build-id=none and --strip-all. It is needed for // reproducible builds; removing references to temporary files in C-land, and - // making build-id reproducably absent. + // making build-id reproducibly absent. extld := []string{"-Wl,-z,stack-size=0x800000,--build-id=none,--strip-all"} if staticLinking { extld = append(extld, "-static") diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index f80dd02c67..c0623a69bf 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -201,17 +201,16 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 { misc.ApplyDAOHardFork(statedb) } + evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig) if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { - evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if pre.Env.BlockHashes != nil && chainConfig.IsPrague(new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) { var ( prevNumber = pre.Env.Number - 1 prevHash = pre.Env.BlockHashes[math.HexOrDecimal64(prevNumber)] - evm = vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) ) - core.ProcessParentBlockHash(prevHash, evm, statedb) + core.ProcessParentBlockHash(prevHash, evm) } for i := 0; txIt.Next(); i++ { tx, err := txIt.Tx() @@ -246,8 +245,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, if err != nil { return nil, nil, nil, err } + // TODO (rjl493456442) it's a bit weird to reset the tracer in the + // middle of block execution, please improve it somehow. if tracer != nil { - vmConfig.Tracer = tracer.Hooks + evm.SetTracer(tracer.Hooks) } statedb.SetTxContext(tx.Hash(), txIndex) @@ -256,12 +257,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, snapshot = statedb.Snapshot() prevGas = gaspool.Gas() ) - evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) - if tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } // (ret []byte, usedGas uint64, failed bool, err error) + + evm.SetTxContext(txContext) msgResult, err := core.ApplyMessage(evm, msg, gaspool) if err != nil { statedb.RevertToSnapshot(snapshot) @@ -375,12 +376,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err)) } requests = append(requests, depositRequests) - // create EVM for system calls - vmenv := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vm.Config{}) + // EIP-7002 withdrawals - requests = append(requests, core.ProcessWithdrawalQueue(vmenv, statedb)) + requests = append(requests, core.ProcessWithdrawalQueue(evm)) // EIP-7251 consolidations - requests = append(requests, core.ProcessConsolidationQueue(vmenv, statedb)) + requests = append(requests, core.ProcessConsolidationQueue(evm)) } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index 586979e772..e679a9e557 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -98,11 +98,8 @@ func (b *BlockGen) Difficulty() *big.Int { // block. func (b *BlockGen) SetParentBeaconRoot(root common.Hash) { b.header.ParentBeaconRoot = &root - var ( - blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) - vmenv = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.cm.config, vm.Config{}) - ) - ProcessBeaconBlockRoot(root, vmenv, b.statedb) + blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) + ProcessBeaconBlockRoot(root, vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{})) } // addTx adds a transaction to the generated block. If no coinbase has @@ -116,8 +113,12 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti if b.gasPool == nil { b.SetCoinbase(common.Address{}) } + var ( + blockContext = NewEVMBlockContext(b.header, bc, &b.header.Coinbase) + evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) + ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(b.cm.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig) + receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) if err != nil { panic(err) } @@ -360,12 +361,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse requests = append(requests, depositRequests) // create EVM for system calls blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, cm.config, vm.Config{}) + evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(vmenv, statedb) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(vmenv, statedb) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -466,8 +467,8 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine if config.IsPrague(b.header.Number, b.header.Time) { // EIP-2935 blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, vmenv, statedb) + evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) + ProcessParentBlockHash(b.header.ParentHash, evm) } // Execute any user modifications to the block. diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 8a0fd1989a..4b0774f2ae 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -50,16 +50,6 @@ type ( leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) ) -// GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. -func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) -} - -// GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. -func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true) -} - // GenerateTrie takes the whole snapshot tree as the input, traverses all the // accounts as well as the corresponding storages and regenerate the whole state // (account trie + all storage tries). diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 779c1ea98c..dce4f79a11 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" + "golang.org/x/exp/maps" ) var ( @@ -73,23 +74,14 @@ var ( // bloom key for an account/slot. This is randomized at init(), so that the // global population of nodes do not all display the exact same behaviour with // regards to bloom content - bloomDestructHasherOffset = 0 - bloomAccountHasherOffset = 0 - bloomStorageHasherOffset = 0 + bloomAccountHasherOffset = 0 + bloomStorageHasherOffset = 0 ) func init() { // Init the bloom offsets in the range [0:24] (requires 8 bytes) - bloomDestructHasherOffset = rand.Intn(25) bloomAccountHasherOffset = rand.Intn(25) bloomStorageHasherOffset = rand.Intn(25) - - // The destruct and account blooms must be different, as the storage slots - // will check for destruction too for every bloom miss. It should not collide - // with modified accounts. - for bloomAccountHasherOffset == bloomDestructHasherOffset { - bloomAccountHasherOffset = rand.Intn(25) - } } // diffLayer represents a collection of modifications made to a state snapshot @@ -106,29 +98,16 @@ type diffLayer struct { root common.Hash // Root hash to which this snapshot diff belongs to stale atomic.Bool // Signals that the layer became stale (state progressed) - // destructSet is a very special helper marker. If an account is marked as - // deleted, then it's recorded in this set. However it's allowed that an account - // is included here but still available in other sets(e.g. storageData). The - // reason is the diff layer includes all the changes in a *block*. It can - // happen that in the tx_1, account A is self-destructed while in the tx_2 - // it's recreated. But we still need this marker to indicate the "old" A is - // deleted, all data in other set belongs to the "new" A. - destructSet map[common.Hash]struct{} // Keyed markers for deleted (and potentially) recreated accounts - accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) - storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) + accountList []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil + storageList map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer lock sync.RWMutex } -// destructBloomHash is used to convert a destruct event into a 64 bit mini hash. -func destructBloomHash(h common.Hash) uint64 { - return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8]) -} - // accountBloomHash is used to convert an account hash into a 64 bit mini hash. func accountBloomHash(h common.Hash) uint64 { return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8]) @@ -142,12 +121,11 @@ func storageBloomHash(h0, h1 common.Hash) uint64 { // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low // level persistent database or a hierarchical diff already. -func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { +func newDiffLayer(parent snapshot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { // Create the new layer with some pre-allocated data segments dl := &diffLayer{ parent: parent, root: root, - destructSet: destructs, accountData: accounts, storageData: storage, storageList: make(map[common.Hash][]common.Hash), @@ -161,10 +139,7 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s panic("unknown parent type") } // Sanity check that accounts or storage slots are never nil - for accountHash, blob := range accounts { - if blob == nil { - panic(fmt.Sprintf("account %#x nil", accountHash)) - } + for _, blob := range accounts { // Determine memory size and track the dirty writes dl.memory += uint64(common.HashLength + len(blob)) snapshotDirtyAccountWriteMeter.Mark(int64(len(blob))) @@ -179,7 +154,6 @@ func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]s snapshotDirtyStorageWriteMeter.Mark(int64(len(data))) } } - dl.memory += uint64(len(destructs) * common.HashLength) return dl } @@ -204,10 +178,6 @@ func (dl *diffLayer) rebloom(origin *diskLayer) { } else { dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs)) } - // Iterate over all the accounts and storage slots and index them - for hash := range dl.destructSet { - dl.diffed.AddHash(destructBloomHash(hash)) - } for hash := range dl.accountData { dl.diffed.AddHash(accountBloomHash(hash)) } @@ -274,11 +244,8 @@ func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) { } // Check the bloom filter first whether there's even a point in reaching into // all the maps in all the layers below - hit := dl.diffed.ContainsHash(accountBloomHash(hash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(hash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(accountBloomHash(hash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -310,18 +277,14 @@ func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) { if data, ok := dl.accountData[hash]; ok { snapshotDirtyAccountHitMeter.Mark(1) snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountReadMeter.Mark(int64(len(data))) + if n := len(data); n > 0 { + snapshotDirtyAccountReadMeter.Mark(int64(n)) + } else { + snapshotDirtyAccountInexMeter.Mark(1) + } snapshotBloomAccountTrueHitMeter.Mark(1) return data, nil } - // If the account is known locally, but deleted, return it - if _, ok := dl.destructSet[hash]; ok { - snapshotDirtyAccountHitMeter.Mark(1) - snapshotDirtyAccountHitDepthHist.Update(int64(depth)) - snapshotDirtyAccountInexMeter.Mark(1) - snapshotBloomAccountTrueHitMeter.Mark(1) - return nil, nil - } // Account unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.accountRLP(hash, depth+1) @@ -345,11 +308,8 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro dl.lock.RUnlock() return nil, ErrSnapshotStale } - hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) - if !hit { - hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) - } var origin *diskLayer + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) if !hit { origin = dl.origin // extract origin while holding the lock } @@ -391,14 +351,6 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ return data, nil } } - // If the account is known locally, but deleted, return an empty slot - if _, ok := dl.destructSet[accountHash]; ok { - snapshotDirtyStorageHitMeter.Mark(1) - snapshotDirtyStorageHitDepthHist.Update(int64(depth)) - snapshotDirtyStorageInexMeter.Mark(1) - snapshotBloomStorageTrueHitMeter.Mark(1) - return nil, nil - } // Storage slot unknown to this diff, resolve from parent if diff, ok := dl.parent.(*diffLayer); ok { return diff.storage(accountHash, storageHash, depth+1) @@ -410,8 +362,8 @@ func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([ // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. -func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockRoot, destructs, accounts, storage) +func (dl *diffLayer) Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockRoot, accounts, storage) } // flatten pushes all data from this point downwards, flattening everything into @@ -436,12 +388,6 @@ func (dl *diffLayer) flatten() snapshot { if parent.stale.Swap(true) { panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo } - // Overwrite all the updated accounts blindly, merge the sorted list - for hash := range dl.destructSet { - parent.destructSet[hash] = struct{}{} - delete(parent.accountData, hash) - delete(parent.storageData, hash) - } for hash, data := range dl.accountData { parent.accountData[hash] = data } @@ -453,17 +399,13 @@ func (dl *diffLayer) flatten() snapshot { continue } // Storage exists in both parent and child, merge the slots - comboData := parent.storageData[accountHash] - for storageHash, data := range storage { - comboData[storageHash] = data - } + maps.Copy(parent.storageData[accountHash], storage) } // Return the combo parent return &diffLayer{ parent: parent.parent, origin: parent.origin, root: dl.root, - destructSet: parent.destructSet, accountData: parent.accountData, storageData: parent.storageData, storageList: make(map[common.Hash][]common.Hash), @@ -489,15 +431,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.lock.Lock() defer dl.lock.Unlock() - dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData)) - for hash := range dl.accountData { - dl.accountList = append(dl.accountList, hash) - } - for hash := range dl.destructSet { - if _, ok := dl.accountData[hash]; !ok { - dl.accountList = append(dl.accountList, hash) - } - } + dl.accountList = maps.Keys(dl.accountData) slices.SortFunc(dl.accountList, common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList @@ -512,18 +446,17 @@ func (dl *diffLayer) AccountList() []common.Hash { // not empty but the flag is true. // // Note, the returned slice is not a copy, so do not modify it. -func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) { +func (dl *diffLayer) StorageList(accountHash common.Hash) []common.Hash { dl.lock.RLock() - _, destructed := dl.destructSet[accountHash] if _, ok := dl.storageData[accountHash]; !ok { // Account not tracked by this layer dl.lock.RUnlock() - return nil, destructed + return nil } // If an old list already exists, return it if list, exist := dl.storageList[accountHash]; exist { dl.lock.RUnlock() - return list, destructed // the cached list can't be nil + return list // the cached list can't be nil } dl.lock.RUnlock() @@ -531,13 +464,9 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) dl.lock.Lock() defer dl.lock.Unlock() - storageMap := dl.storageData[accountHash] - storageList := make([]common.Hash, 0, len(storageMap)) - for k := range storageMap { - storageList = append(storageList, k) - } + storageList := maps.Keys(dl.storageData[accountHash]) slices.SortFunc(storageList, common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) - return storageList, destructed + return storageList } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 674a031b16..b212098e75 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -28,14 +28,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { - copy := make(map[common.Hash]struct{}) - for hash := range destructs { - copy[hash] = struct{}{} - } - return copy -} - func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte { copy := make(map[common.Hash][]byte) for hash, blob := range accounts { @@ -58,9 +50,8 @@ func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash // TestMergeBasics tests some simple merges func TestMergeBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -69,7 +60,7 @@ func TestMergeBasics(t *testing.T) { accounts[h] = data if rand.Intn(4) == 0 { - destructs[h] = struct{}{} + accounts[h] = nil } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) @@ -80,11 +71,12 @@ func TestMergeBasics(t *testing.T) { } } // Add some (identical) layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) - child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + parent := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child := newDiffLayer(parent, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + child = newDiffLayer(child, common.Hash{}, copyAccounts(accounts), copyStorage(storage)) + // And flatten merged := (child.flatten()).(*diffLayer) @@ -99,18 +91,13 @@ func TestMergeBasics(t *testing.T) { t.Errorf("accountList [2] wrong: have %v, want %v", have, want) } } - { // Check account drops - if have, want := len(merged.destructSet), len(destructs); have != want { - t.Errorf("accountDrop wrong: have %v, want %v", have, want) - } - } { // Check storage lists i := 0 for aHash, sMap := range storage { if have, want := len(merged.storageList), i; have != want { t.Errorf("[1] storageList wrong: have %v, want %v", have, want) } - list, _ := merged.StorageList(aHash) + list := merged.StorageList(aHash) if have, want := len(list), len(sMap); have != want { t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want) } @@ -124,41 +111,32 @@ func TestMergeBasics(t *testing.T) { // TestMergeDelete tests some deletion func TestMergeDelete(t *testing.T) { - var ( - storage = make(map[common.Hash]map[common.Hash][]byte) - ) + storage := make(map[common.Hash]map[common.Hash][]byte) + // Fill up a parent h1 := common.HexToHash("0x01") h2 := common.HexToHash("0x02") - flipDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h2: {}, - } - } - flipAccs := func() map[common.Hash][]byte { + flip := func() map[common.Hash][]byte { return map[common.Hash][]byte{ h1: randomAccount(), + h2: nil, } } - flopDrops := func() map[common.Hash]struct{} { - return map[common.Hash]struct{}{ - h1: {}, - } - } - flopAccs := func() map[common.Hash][]byte { + flop := func() map[common.Hash][]byte { return map[common.Hash][]byte{ + h1: nil, h2: randomAccount(), } } // Add some flipAccs-flopping layers on top - parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage) - child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) - child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage) - child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage) + parent := newDiffLayer(emptyLayer(), common.Hash{}, flip(), storage) + child := parent.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) + child = child.Update(common.Hash{}, flop(), storage) + child = child.Update(common.Hash{}, flip(), storage) if data, _ := child.Account(h1); data == nil { t.Errorf("last diff layer: expected %x account to be non-nil", h1) @@ -166,12 +144,7 @@ func TestMergeDelete(t *testing.T) { if data, _ := child.Account(h2); data != nil { t.Errorf("last diff layer: expected %x account to be nil", h2) } - if _, ok := child.destructSet[h1]; ok { - t.Errorf("last diff layer: expected %x drop to be missing", h1) - } - if _, ok := child.destructSet[h2]; !ok { - t.Errorf("last diff layer: expected %x drop to be present", h1) - } + // And flatten merged := (child.flatten()).(*diffLayer) @@ -181,12 +154,6 @@ func TestMergeDelete(t *testing.T) { if data, _ := merged.Account(h2); data != nil { t.Errorf("merged layer: expected %x account to be nil", h2) } - if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } - if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk! - t.Errorf("merged diff layer: expected %x drop to be present", h1) - } // If we add more granular metering of memory, we can enable this again, // but it's not implemented for now //if have, want := merged.memory, child.memory; have != want { @@ -206,22 +173,20 @@ func TestInsertAndMerge(t *testing.T) { ) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) - parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage) + parent = newDiffLayer(emptyLayer(), common.Hash{}, accounts, storage) } { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[acc] = randomAccount() storage[acc] = make(map[common.Hash][]byte) storage[acc][slot] = []byte{0x01} - child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + child = newDiffLayer(parent, common.Hash{}, accounts, storage) } // And flatten merged := (child.flatten()).(*diffLayer) @@ -250,14 +215,13 @@ func BenchmarkSearch(b *testing.B) { // First, we set up 128 diff layers, with 1K items each fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 10000; i++ { accounts[randomHash()] = randomAccount() } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -286,9 +250,8 @@ func BenchmarkSearchSlot(b *testing.B) { accountRLP := randomAccount() fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) accounts[accountKey] = accountRLP @@ -299,7 +262,7 @@ func BenchmarkSearchSlot(b *testing.B) { accStorage[randomHash()] = value storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } var layer snapshot layer = emptyLayer() @@ -320,9 +283,8 @@ func BenchmarkSearchSlot(b *testing.B) { func BenchmarkFlatten(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 100; i++ { accountKey := randomHash() @@ -336,7 +298,7 @@ func BenchmarkFlatten(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } b.ResetTimer() for i := 0; i < b.N; i++ { @@ -369,9 +331,8 @@ func BenchmarkFlatten(b *testing.B) { func BenchmarkJournal(b *testing.B) { fill := func(parent snapshot) *diffLayer { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) for i := 0; i < 200; i++ { accountKey := randomHash() @@ -385,7 +346,7 @@ func BenchmarkJournal(b *testing.B) { } storage[accountKey] = accStorage } - return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage) + return newDiffLayer(parent, common.Hash{}, accounts, storage) } layer := snapshot(emptyLayer()) for i := 1; i < 128; i++ { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 76928edf07..202e6c70ed 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -180,8 +180,8 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. Note, the maps are retained by the method to avoid // copying everything. -func (dl *diskLayer) Update(blockHash common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { - return newDiffLayer(dl, blockHash, destructs, accounts, storage) +func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer { + return newDiffLayer(dl, blockHash, accounts, storage) } // stopGeneration aborts the state snapshot generation if it is currently running. diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 168458c405..6d99a90d61 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -117,20 +117,22 @@ func TestDiskMerge(t *testing.T) { base.Storage(conNukeCache, conNukeCacheSlot) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: {conNukeNoCacheSlot: nil}, + conNukeCache: {conNukeCacheSlot: nil}, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -340,20 +342,27 @@ func TestDiskPartialMerge(t *testing.T) { assertStorage(conNukeCache, conNukeCacheSlot, conNukeCacheSlot[:]) // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, map[common.Hash]struct{}{ - accDelNoCache: {}, - accDelCache: {}, - conNukeNoCache: {}, - conNukeCache: {}, - }, map[common.Hash][]byte{ - accModNoCache: reverse(accModNoCache[:]), - accModCache: reverse(accModCache[:]), - }, map[common.Hash]map[common.Hash][]byte{ - conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, - conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, - conDelNoCache: {conDelNoCacheSlot: nil}, - conDelCache: {conDelCacheSlot: nil}, - }); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accDelNoCache: nil, + accDelCache: nil, + conNukeNoCache: nil, + conNukeCache: nil, + accModNoCache: reverse(accModNoCache[:]), + accModCache: reverse(accModCache[:]), + }, + map[common.Hash]map[common.Hash][]byte{ + conNukeNoCache: { + conNukeNoCacheSlot: nil, + }, + conNukeCache: { + conNukeCacheSlot: nil, + }, + conModNoCache: {conModNoCacheSlot: reverse(conModNoCacheSlot[:])}, + conModCache: {conModCacheSlot: reverse(conModCacheSlot[:])}, + conDelNoCache: {conDelNoCacheSlot: nil}, + conDelCache: {conDelCacheSlot: nil}, + }); err != nil { t.Fatalf("test %d: failed to update snapshot tree: %v", i, err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -462,9 +471,11 @@ func TestDiskGeneratorPersistence(t *testing.T) { }, } // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffRoot, baseRoot, nil, map[common.Hash][]byte{ - accTwo: accTwo[:], - }, nil); err != nil { + if err := snaps.Update(diffRoot, baseRoot, + map[common.Hash][]byte{ + accTwo: accTwo[:], + }, nil, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } if err := snaps.Cap(diffRoot, 0); err != nil { @@ -480,11 +491,14 @@ func TestDiskGeneratorPersistence(t *testing.T) { } // Test scenario 2, the disk layer is fully generated // Modify or delete some accounts, flatten everything onto disk - if err := snaps.Update(diffTwoRoot, diffRoot, nil, map[common.Hash][]byte{ - accThree: accThree.Bytes(), - }, map[common.Hash]map[common.Hash][]byte{ - accThree: {accThreeSlot: accThreeSlot.Bytes()}, - }); err != nil { + if err := snaps.Update(diffTwoRoot, diffRoot, + map[common.Hash][]byte{ + accThree: accThree.Bytes(), + }, + map[common.Hash]map[common.Hash][]byte{ + accThree: {accThreeSlot: accThreeSlot.Bytes()}, + }, + ); err != nil { t.Fatalf("failed to update snapshot tree: %v", err) } diskLayer := snaps.layers[snaps.diskRoot()].(*diskLayer) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 56abff348d..661610840a 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -134,7 +134,7 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { - storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) + storageIt := snap.StorageIterator(accountHash, common.Hash{}) defer storageIt.Release() hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index c1a196c7ff..cd9c225844 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -115,6 +115,7 @@ func (it *diffAccountIterator) Next() bool { } // Iterator seems to be still alive, retrieve and cache the live hash it.curHash = it.keys[0] + // key cached, shift the iterator and notify the user of success it.keys = it.keys[1:] return true @@ -135,7 +136,7 @@ func (it *diffAccountIterator) Hash() common.Hash { // This method may _fail_, if the underlying layer has been flattened between // the call to Next and Account. That type of error will set it.Err. // This method assumes that flattening does not delete elements from -// the accountdata mapping (writing nil into it is fine though), and will panic +// the accountData mapping (writing nil into it is fine though), and will panic // if elements have been deleted. // // Note the returned account is not a copy, please don't modify it. @@ -143,10 +144,6 @@ func (it *diffAccountIterator) Account() []byte { it.layer.lock.RLock() blob, ok := it.layer.accountData[it.curHash] if !ok { - if _, ok := it.layer.destructSet[it.curHash]; ok { - it.layer.lock.RUnlock() - return nil - } panic(fmt.Sprintf("iterator referenced non-existent account: %x", it.curHash)) } it.layer.lock.RUnlock() @@ -247,11 +244,11 @@ type diffStorageIterator struct { // "destructed" returned. If it's true then it means the whole storage is // destructed in this layer(maybe recreated too), don't bother deeper layer // for storage retrieval. -func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { // Create the storage for this account even it's marked // as destructed. The iterator is for the new one which // just has the same address as the deleted one. - hashes, destructed := dl.StorageList(account) + hashes := dl.StorageList(account) index := sort.Search(len(hashes), func(i int) bool { return bytes.Compare(seek[:], hashes[i][:]) <= 0 }) @@ -260,7 +257,7 @@ func (dl *diffLayer) StorageIterator(account common.Hash, seek common.Hash) (Sto layer: dl, account: account, keys: hashes[index:], - }, destructed + } } // Next steps the iterator forward one element, returning false if exhausted. @@ -339,13 +336,13 @@ type diskStorageIterator struct { // If the whole storage is destructed, then all entries in the disk // layer are deleted already. So the "destructed" flag returned here // is always false. -func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) { +func (dl *diskLayer) StorageIterator(account common.Hash, seek common.Hash) StorageIterator { pos := common.TrimRightZeroes(seek[:]) return &diskStorageIterator{ layer: dl, account: account, it: dl.diskdb.NewIterator(append(rawdb.SnapshotStoragePrefix, account.Bytes()...), pos), - }, false + } } // Next steps the iterator forward one element, returning false if exhausted. diff --git a/core/state/snapshot/iterator_binary.go b/core/state/snapshot/iterator_binary.go index 523c7b9946..254c4b860f 100644 --- a/core/state/snapshot/iterator_binary.go +++ b/core/state/snapshot/iterator_binary.go @@ -67,44 +67,17 @@ func (dl *diffLayer) initBinaryAccountIterator(seek common.Hash) Iterator { func (dl *diffLayer) initBinaryStorageIterator(account, seek common.Hash) Iterator { parent, ok := dl.parent.(*diffLayer) if !ok { - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } - // The parent is disk layer, don't need to take care "destructed" - // anymore. - b, _ := dl.Parent().StorageIterator(account, seek) l := &binaryIterator{ - a: a, - b: b, + a: dl.StorageIterator(account, seek), + b: dl.Parent().StorageIterator(account, seek), account: account, } l.aDone = !l.a.Next() l.bDone = !l.b.Next() return l } - // If the storage in this layer is already destructed, discard all - // deeper layers but still return a valid single-branch iterator. - a, destructed := dl.StorageIterator(account, seek) - if destructed { - l := &binaryIterator{ - a: a, - account: account, - } - l.aDone = !l.a.Next() - l.bDone = true - return l - } l := &binaryIterator{ - a: a, + a: dl.StorageIterator(account, seek), b: parent.initBinaryStorageIterator(account, seek), account: account, } diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index fa0daea7ba..7f7ba876ff 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -90,18 +90,10 @@ func newFastIterator(tree *Tree, root common.Hash, account common.Hash, seek com priority: depth, }) } else { - // If the whole storage is destructed in this layer, don't - // bother deeper layer anymore. But we should still keep - // the iterator for this layer, since the iterator can contain - // some valid slots which belongs to the re-created account. - it, destructed := current.StorageIterator(account, seek) fi.iterators = append(fi.iterators, &weightedIterator{ - it: it, + it: current.StorageIterator(account, seek), priority: depth, }) - if destructed { - break - } } current = current.Parent() } diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index f2f4d19f83..b9fe370b86 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -32,9 +32,8 @@ import ( // TestAccountIteratorBasics tests some simple single-layer(diff and disk) iteration func TestAccountIteratorBasics(t *testing.T) { var ( - destructs = make(map[common.Hash]struct{}) - accounts = make(map[common.Hash][]byte) - storage = make(map[common.Hash]map[common.Hash][]byte) + accounts = make(map[common.Hash][]byte) + storage = make(map[common.Hash]map[common.Hash][]byte) ) // Fill up a parent for i := 0; i < 100; i++ { @@ -42,9 +41,6 @@ func TestAccountIteratorBasics(t *testing.T) { data := randomAccount() accounts[h] = data - if rand.Intn(4) == 0 { - destructs[h] = struct{}{} - } if rand.Intn(2) == 0 { accStorage := make(map[common.Hash][]byte) value := make([]byte, 32) @@ -54,7 +50,7 @@ func TestAccountIteratorBasics(t *testing.T) { } } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) it := diffLayer.AccountIterator(common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator @@ -95,15 +91,15 @@ func TestStorageIteratorBasics(t *testing.T) { nilStorage[h] = nilstorage } // Add some (identical) layers on top - diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, nil, copyAccounts(accounts), copyStorage(storage)) + diffLayer := newDiffLayer(emptyLayer(), common.Hash{}, copyAccounts(accounts), copyStorage(storage)) for account := range accounts { - it, _ := diffLayer.StorageIterator(account, common.Hash{}) + it := diffLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100, it, verifyNothing) // Nil is allowed for single layer iterator } diskLayer := diffToDisk(diffLayer) for account := range accounts { - it, _ := diskLayer.StorageIterator(account, common.Hash{}) + it := diskLayer.StorageIterator(account, common.Hash{}) verifyIterator(t, 100-nilStorage[account], it, verifyNothing) // Nil is allowed for single layer iterator } } @@ -225,13 +221,13 @@ func TestAccountIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Verify the single and multi-layer iterators @@ -272,19 +268,19 @@ func TestStorageIteratorTraversal(t *testing.T) { }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x04", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x02", "0x03"}}, nil)) // Verify the single and multi-layer iterators head := snaps.Snapshot(common.HexToHash("0x04")) - diffIter, _ := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) + diffIter := head.(snapshot).StorageIterator(common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 3, diffIter, verifyNothing) verifyIterator(t, 6, head.(*diffLayer).newBinaryStorageIterator(common.HexToHash("0xaa"), common.Hash{}), verifyStorage) @@ -357,14 +353,14 @@ func TestAccountIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, a, nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, b, nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, c, nil) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, d, nil) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, e, nil) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, f, nil) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, g, nil) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, h, nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), a, nil) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), b, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), c, nil) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), d, nil) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), e, nil) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), f, nil) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), g, nil) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), h, nil) it, _ := snaps.AccountIterator(common.HexToHash("0x09"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -456,14 +452,14 @@ func TestStorageIteratorTraversalValues(t *testing.T) { } } // Assemble a stack of snapshots from the account layers - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, randomAccountSet("0xaa"), wrapStorage(a)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, randomAccountSet("0xaa"), wrapStorage(b)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, randomAccountSet("0xaa"), wrapStorage(c)) - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, randomAccountSet("0xaa"), wrapStorage(d)) - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), nil, randomAccountSet("0xaa"), wrapStorage(e)) - snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), nil, randomAccountSet("0xaa"), wrapStorage(g)) - snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), nil, randomAccountSet("0xaa"), wrapStorage(h)) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), wrapStorage(a)) + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), wrapStorage(b)) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), wrapStorage(c)) + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), wrapStorage(d)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x07"), common.HexToHash("0x06"), randomAccountSet("0xaa"), wrapStorage(e)) + snaps.Update(common.HexToHash("0x08"), common.HexToHash("0x07"), randomAccountSet("0xaa"), wrapStorage(g)) + snaps.Update(common.HexToHash("0x09"), common.HexToHash("0x08"), randomAccountSet("0xaa"), wrapStorage(h)) it, _ := snaps.StorageIterator(common.HexToHash("0x09"), common.HexToHash("0xaa"), common.Hash{}) head := snaps.Snapshot(common.HexToHash("0x09")) @@ -526,7 +522,7 @@ func TestAccountIteratorLargeTraversal(t *testing.T) { }, } for i := 1; i < 128; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // Iterate the entire stack and ensure everything is hit only once head := snaps.Snapshot(common.HexToHash("0x80")) @@ -584,13 +580,13 @@ func testAccountIteratorFlattening(t *testing.T, newIterator func(snaps *Tree, r }, } // Create a stack of diffs on top - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Create an iterator and flatten the data from underneath it @@ -630,13 +626,13 @@ func testAccountIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, s base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa", "0xee", "0xff", "0xf0"), nil) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xbb", "0xdd", "0xf0"), nil) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xcc", "0xf0", "0xff"), nil) // Account set is now @@ -708,13 +704,13 @@ func testStorageIteratorSeek(t *testing.T, newIterator func(snaps *Tree, root, a }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x05", "0x06"}}, nil)) - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x05", "0x08"}}, nil)) // Account set is now @@ -785,21 +781,16 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), - nil, randomAccountSet("0x11", "0x22", "0x33"), nil) - - deleted := common.HexToHash("0x22") - destructed := map[common.Hash]struct{}{ - deleted: {}, - } - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), - destructed, randomAccountSet("0x11", "0x33"), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0x11", "0x22", "0x33"), nil) + set := randomAccountSet("0x11", "0x33") + set[common.HexToHash("0x22")] = nil + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), set, nil) snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), - nil, randomAccountSet("0x33", "0x44", "0x55"), nil) + randomAccountSet("0x33", "0x44", "0x55"), nil) // The output should be 11,33,44,55 - it := newIterator(snaps, common.HexToHash("0x04"), (common.Hash{})) + it := newIterator(snaps, common.HexToHash("0x04"), common.Hash{}) // Do a quick check verifyIterator(t, 4, it, verifyAccount) @@ -813,8 +804,8 @@ func testAccountIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro if it.Account() == nil { t.Errorf("iterator returned nil-value for hash %x", hash) } - if hash == deleted { - t.Errorf("expected deleted elem %x to not be returned by iterator", deleted) + if hash == common.HexToHash("0x22") { + t.Errorf("expected deleted elem %x to not be returned by iterator", common.HexToHash("0x22")) } } } @@ -846,10 +837,10 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro }, } // Stack three diff layers on top with various overlaps - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x01", "0x03", "0x05"}}, nil)) - snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, + snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x02", "0x04", "0x06"}}, [][]string{{"0x01", "0x03"}})) // The output should be 02,04,05,06 @@ -863,17 +854,16 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage - destructed := map[common.Hash]struct{}{ - common.HexToHash("0xaa"): {}, - } - snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), destructed, nil, nil) + snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), + map[common.Hash][]byte{common.HexToHash("0xaa"): nil}, + randomStorageSet([]string{"0xaa"}, nil, [][]string{{"0x02", "0x04", "0x05", "0x06"}})) it = newIterator(snaps, common.HexToHash("0x04"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 0, it, verifyStorage) it.Release() // Re-insert the slots of the same account - snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), nil, + snaps.Update(common.HexToHash("0x05"), common.HexToHash("0x04"), randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x07", "0x08", "0x09"}}, nil)) // The output should be 07,08,09 @@ -883,7 +873,9 @@ func testStorageIteratorDeletions(t *testing.T, newIterator func(snaps *Tree, ro it.Release() // Destruct the whole storage but re-create the account in the same layer - snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), destructed, randomAccountSet("0xaa"), randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, nil)) + snaps.Update(common.HexToHash("0x06"), common.HexToHash("0x05"), + randomAccountSet("0xaa"), + randomStorageSet([]string{"0xaa"}, [][]string{{"0x11", "0x12"}}, [][]string{{"0x07", "0x08", "0x09"}})) it = newIterator(snaps, common.HexToHash("0x06"), common.HexToHash("0xaa"), common.Hash{}) verifyIterator(t, 2, it, verifyStorage) // The output should be 11,12 it.Release() @@ -928,7 +920,7 @@ func BenchmarkAccountIteratorTraversal(b *testing.B) { }, } for i := 1; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(200), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(200), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. @@ -1023,9 +1015,9 @@ func BenchmarkAccountIteratorLargeBaselayer(b *testing.B) { base.root: base, }, } - snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, makeAccounts(2000), nil) + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(2000), nil) for i := 2; i <= 100; i++ { - snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), nil, makeAccounts(20), nil) + snaps.Update(common.HexToHash(fmt.Sprintf("0x%02x", i+1)), common.HexToHash(fmt.Sprintf("0x%02x", i)), makeAccounts(20), nil) } // We call this once before the benchmark, so the creation of // sorted accountlists are not included in the results. diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 8513e73dd0..61552c73fe 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -33,7 +33,9 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -const journalVersion uint64 = 0 +// 0: initial version +// 1: destruct flag in diff layer is removed +const journalVersion uint64 = 1 // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { @@ -48,11 +50,6 @@ type journalGenerator struct { Storage uint64 } -// journalDestruct is an account deletion entry in a diffLayer's disk journal. -type journalDestruct struct { - Hash common.Hash -} - // journalAccount is an account entry in a diffLayer's disk journal. type journalAccount struct { Hash common.Hash @@ -109,8 +106,8 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. var current snapshot = base - err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { - current = newDiffLayer(current, root, destructSet, accountData, storageData) + err := iterateJournal(db, func(parent common.Hash, root common.Hash, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, accountData, storageData) return nil }) if err != nil { @@ -238,16 +235,12 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { if err := rlp.Encode(buffer, dl.root); err != nil { return common.Hash{}, err } - destructs := make([]journalDestruct, 0, len(dl.destructSet)) - for hash := range dl.destructSet { - destructs = append(destructs, journalDestruct{Hash: hash}) - } - if err := rlp.Encode(buffer, destructs); err != nil { - return common.Hash{}, err - } accounts := make([]journalAccount, 0, len(dl.accountData)) for hash, blob := range dl.accountData { - accounts = append(accounts, journalAccount{Hash: hash, Blob: blob}) + accounts = append(accounts, journalAccount{ + Hash: hash, + Blob: blob, + }) } if err := rlp.Encode(buffer, accounts); err != nil { return common.Hash{}, err @@ -271,7 +264,7 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { // journalCallback is a function which is invoked by iterateJournal, every // time a difflayer is loaded from disk. -type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error +type journalCallback = func(parent common.Hash, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error // iterateJournal iterates through the journalled difflayers, loading them from // the database, and invoking the callback for each loaded layer. @@ -310,10 +303,8 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { for { var ( root common.Hash - destructs []journalDestruct accounts []journalAccount storage []journalStorage - destructSet = make(map[common.Hash]struct{}) accountData = make(map[common.Hash][]byte) storageData = make(map[common.Hash]map[common.Hash][]byte) ) @@ -325,18 +316,12 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } return fmt.Errorf("load diff root: %v", err) } - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) - } if err := r.Decode(&accounts); err != nil { return fmt.Errorf("load diff accounts: %v", err) } if err := r.Decode(&storage); err != nil { return fmt.Errorf("load diff storage: %v", err) } - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } for _, entry := range accounts { if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that accountData[entry.Hash] = entry.Blob @@ -355,7 +340,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } storageData[entry.Hash] = slots } - if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + if err := callback(parent, root, accountData, storageData); err != nil { return err } parent = root diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index d1ffb5cc2d..181dc6b746 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -130,7 +130,7 @@ type snapshot interface { // the specified data items. // // Note, the maps are retained by the method to avoid copying everything. - Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer + Update(blockRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) *diffLayer // Journal commits an entire diff hierarchy to disk into a single journal entry. // This is meant to be used during shutdown to persist the snapshot without @@ -145,7 +145,7 @@ type snapshot interface { AccountIterator(seek common.Hash) AccountIterator // StorageIterator creates a storage iterator over an arbitrary layer. - StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) + StorageIterator(account common.Hash, seek common.Hash) StorageIterator } // Config includes the configurations for snapshots. @@ -206,47 +206,24 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, roo log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil } - // Create the building waiter iff the background generation is allowed - if !config.NoBuild && !config.AsyncBuild { - defer snap.waitBuild() - } if err != nil { log.Warn("Failed to load snapshot", "err", err) - if !config.NoBuild { - snap.Rebuild(root) - return snap, nil + if config.NoBuild { + return nil, err } - return nil, err // Bail out the error, don't rebuild automatically. + wait := snap.Rebuild(root) + if !config.AsyncBuild { + wait() + } + return snap, nil } // Existing snapshot loaded, seed all the layers - for head != nil { + for ; head != nil; head = head.Parent() { snap.layers[head.Root()] = head - head = head.Parent() } return snap, nil } -// waitBuild blocks until the snapshot finishes rebuilding. This method is meant -// to be used by tests to ensure we're testing what we believe we are. -func (t *Tree) waitBuild() { - // Find the rebuild termination channel - var done chan struct{} - - t.lock.RLock() - for _, layer := range t.layers { - if layer, ok := layer.(*diskLayer); ok { - done = layer.genPending - break - } - } - t.lock.RUnlock() - - // Wait until the snapshot is generated - if done != nil { - <-done - } -} - // Disable interrupts any pending snapshot generator, deletes all the snapshot // layers in memory and marks snapshots disabled globally. In order to resume // the snapshot functionality, the caller must invoke Rebuild. @@ -335,7 +312,7 @@ func (t *Tree) Snapshots(root common.Hash, limits int, nodisk bool) []Snapshot { // Update adds a new snapshot into the tree, if that can be linked to an existing // old parent. It is disallowed to insert a disk layer (the origin of all). -func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { +func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { // Reject noop updates to avoid self-loops in the snapshot tree. This is a // special case that can only happen for Clique networks where empty blocks // don't modify the state (0 block subsidy). @@ -350,7 +327,7 @@ func (t *Tree) Update(blockRoot common.Hash, parentRoot common.Hash, destructs m if parent == nil { return fmt.Errorf("parent [%#x] snapshot missing", parentRoot) } - snap := parent.(snapshot).Update(blockRoot, destructs, accounts, storage) + snap := parent.(snapshot).Update(blockRoot, accounts, storage) // Save the new snapshot for later t.lock.Lock() @@ -539,35 +516,6 @@ func diffToDisk(bottom *diffLayer) *diskLayer { base.stale = true base.lock.Unlock() - // Destroy all the destructed accounts from the database - for hash := range bottom.destructSet { - // Skip any account not covered yet by the snapshot - if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 { - continue - } - // Remove all storage slots - rawdb.DeleteAccountSnapshot(batch, hash) - base.cache.Set(hash[:], nil) - - it := rawdb.IterateStorageSnapshots(base.diskdb, hash) - for it.Next() { - key := it.Key() - batch.Delete(key) - base.cache.Del(key[1:]) - snapshotFlushStorageItemMeter.Mark(1) - - // Ensure we don't delete too much data blindly (contract can be - // huge). It's ok to flush, the root will go missing in case of a - // crash and we'll detect and regenerate the snapshot. - if batch.ValueSize() > 64*1024*1024 { - if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) - } - batch.Reset() - } - } - it.Release() - } // Push all updated accounts into the database for hash, data := range bottom.accountData { // Skip any account not covered yet by the snapshot @@ -575,10 +523,14 @@ func diffToDisk(bottom *diffLayer) *diskLayer { continue } // Push the account to disk - rawdb.WriteAccountSnapshot(batch, hash, data) - base.cache.Set(hash[:], data) - snapshotCleanAccountWriteMeter.Mark(int64(len(data))) - + if len(data) != 0 { + rawdb.WriteAccountSnapshot(batch, hash, data) + base.cache.Set(hash[:], data) + snapshotCleanAccountWriteMeter.Mark(int64(len(data))) + } else { + rawdb.DeleteAccountSnapshot(batch, hash) + base.cache.Set(hash[:], nil) + } snapshotFlushAccountItemMeter.Mark(1) snapshotFlushAccountSizeMeter.Mark(int64(len(data))) @@ -587,7 +539,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // the snapshot. if batch.ValueSize() > 64*1024*1024 { if err := batch.Write(); err != nil { - log.Crit("Failed to write storage deletions", "err", err) + log.Crit("Failed to write state changes", "err", err) } batch.Reset() } @@ -616,6 +568,16 @@ func diffToDisk(bottom *diffLayer) *diskLayer { } snapshotFlushStorageItemMeter.Mark(1) snapshotFlushStorageSizeMeter.Mark(int64(len(data))) + + // Ensure we don't write too much data blindly. It's ok to flush, the + // root will go missing in case of a crash and we'll detect and regen + // the snapshot. + if batch.ValueSize() > 64*1024*1024 { + if err := batch.Write(); err != nil { + log.Crit("Failed to write state changes", "err", err) + } + batch.Reset() + } } } // Update the snapshot block marker and write any remainder data @@ -703,8 +665,9 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { // Rebuild wipes all available snapshot data from the persistent database and // discard all caches and diff layers. Afterwards, it starts a new snapshot -// generator with the given root hash. -func (t *Tree) Rebuild(root common.Hash) { +// generator with the given root hash. The returned function blocks until +// regeneration is complete. +func (t *Tree) Rebuild(root common.Hash) (wait func()) { t.lock.Lock() defer t.lock.Unlock() @@ -736,9 +699,11 @@ func (t *Tree) Rebuild(root common.Hash) { // Start generating a new snapshot from scratch on a background thread. The // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") + disk := generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root) t.layers = map[common.Hash]snapshot{ - root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), + root: disk, } + return func() { <-disk.genPending } } // AccountIterator creates a new account iterator for the specified root hash and diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index a9ab3eaea3..34ef61e8d0 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -107,7 +107,7 @@ func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 2 { @@ -151,10 +151,10 @@ func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 3 { @@ -203,13 +203,13 @@ func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { accounts := map[common.Hash][]byte{ common.HexToHash("0xa1"): randomAccount(), } - if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } - if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { + if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), accounts, nil); err != nil { t.Fatalf("failed to create a diff layer: %v", err) } if n := len(snaps.layers); n != 4 { @@ -263,12 +263,12 @@ func TestPostCapBasicDataAccess(t *testing.T) { }, } // The lowest difflayer - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), setAccount("0xb2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) - snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), setAccount("0xb3"), nil) // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { @@ -363,7 +363,7 @@ func TestSnaphots(t *testing.T) { ) for i := 0; i < 129; i++ { head = makeRoot(uint64(i + 2)) - snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) + snaps.Update(head, last, setAccount(fmt.Sprintf("%d", i+2)), nil) last = head snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) } @@ -456,9 +456,9 @@ func TestReadStateDuringFlattening(t *testing.T) { }, } // 4 layers in total, 3 diff layers and 1 disk layers - snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) - snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) - snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) + snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), setAccount("0xa1"), nil) + snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), setAccount("0xa2"), nil) + snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), setAccount("0xa3"), nil) // Obtain the topmost snapshot handler for state accessing snap := snaps.Snapshot(common.HexToHash("0xa3")) diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index 62f073d2e1..c35c82f67a 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -75,7 +75,7 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { func checkDanglingMemStorage(db ethdb.KeyValueStore) error { start := time.Now() log.Info("Checking dangling journalled storage") - err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + err := iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { for accHash := range storage { if _, ok := accounts[accHash]; !ok { log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) @@ -119,12 +119,11 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { } var depth = 0 - return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + return iterateJournal(db, func(pRoot, root common.Hash, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { _, a := accounts[hash] - _, b := destructs[hash] - _, c := storage[hash] + _, b := storage[hash] depth++ - if !a && !b && !c { + if !a && !b { return nil } fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) @@ -138,9 +137,6 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { fmt.Printf("\taccount.root: %x\n", account.Root) fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - if _, ok := destructs[hash]; ok { - fmt.Printf("\t Destructed!") - } if data, ok := storage[hash]; ok { fmt.Printf("\tStorage\n") for k, v := range data { diff --git a/core/state/statedb.go b/core/state/statedb.go index d855e5626d..9cc91c9332 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -932,16 +932,17 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer iter.Release() var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeleted()) @@ -949,42 +950,47 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, for iter.Next() { slot := common.CopyBytes(iter.Slot()) if err := iter.Error(); err != nil { // error might occur after Slot function - return nil, nil, err + return nil, nil, nil, err } - slots[iter.Hash()] = slot + key := iter.Hash() + storages[key] = nil + storageOrigins[key] = slot - if err := stack.Update(iter.Hash().Bytes(), slot); err != nil { - return nil, nil, err + if err := stack.Update(key.Bytes(), slot); err != nil { + return nil, nil, nil, err } } if err := iter.Error(); err != nil { // error might occur during iteration - return nil, nil, err + return nil, nil, nil, err } if stack.Hash() != root { - return nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) + return nil, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash()) } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage," // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. -func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } it, err := tr.NodeIterator(nil) if err != nil { - return nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + return nil, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) } var ( - nodes = trienode.NewNodeSet(addrHash) - slots = make(map[common.Hash][]byte) + nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil) + storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil) + storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot ) for it.Next(true) { if it.Leaf() { - slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) + key := common.BytesToHash(it.LeafKey()) + storages[key] = nil + storageOrigins[key] = common.CopyBytes(it.LeafBlob()) continue } if it.Hash() == (common.Hash{}) { @@ -993,35 +999,36 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r nodes.AddNode(it.Path(), trienode.NewDeleted()) } if err := it.Error(); err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // deleteStorage is designed to delete the storage trie of a designated account. // The function will make an attempt to utilize an efficient strategy if the // associated state snapshot is reachable; otherwise, it will resort to a less // efficient approach. -func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) { var ( - err error - slots map[common.Hash][]byte - nodes *trienode.NodeSet + err error + nodes *trienode.NodeSet // the set for trie node mutations (value is nil) + storages map[common.Hash][]byte // the set for storage mutations (value is nil) + storageOrigins map[common.Hash][]byte // the set for tracking the original value of slot ) // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. snaps := s.db.Snapshot() if snaps != nil { - slots, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) + storages, storageOrigins, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) } if snaps == nil || err != nil { - slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) + storages, storageOrigins, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return slots, nodes, nil + return storages, storageOrigins, nodes, nil } // handleDestruction processes all destruction markers and deletes the account @@ -1068,16 +1075,16 @@ func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trieno deletes[addrHash] = op // Short circuit if the origin storage was empty. - if prev.Root == types.EmptyRootHash || s.db.TrieDB().IsVerkle() { continue } // Remove storage slots belonging to the account. - slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + storages, storagesOrigin, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err) } - op.storagesOrigin = slots + op.storages = storages + op.storagesOrigin = storagesOrigin // Aggregate the associated trie node changes. nodes = append(nodes, set) @@ -1267,7 +1274,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU // If snapshotting is enabled, update the snapshot tree with this new version if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { + if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 90250819e3..7cbfd9b9d7 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "errors" "fmt" + "maps" "math" "math/rand" "reflect" @@ -176,24 +177,16 @@ func (test *stateTest) String() string { func (test *stateTest) run() bool { var ( - roots []common.Hash - accountList []map[common.Address][]byte - storageList []map[common.Address]map[common.Hash][]byte - copyUpdate = func(update *stateUpdate) { - accounts := make(map[common.Address][]byte, len(update.accountsOrigin)) - for key, val := range update.accountsOrigin { - accounts[key] = common.CopyBytes(val) - } - accountList = append(accountList, accounts) - - storages := make(map[common.Address]map[common.Hash][]byte, len(update.storagesOrigin)) - for addr, subset := range update.storagesOrigin { - storages[addr] = make(map[common.Hash][]byte, len(subset)) - for key, val := range subset { - storages[addr][key] = common.CopyBytes(val) - } - } - storageList = append(storageList, storages) + roots []common.Hash + accounts []map[common.Hash][]byte + accountOrigin []map[common.Address][]byte + storages []map[common.Hash]map[common.Hash][]byte + storageOrigin []map[common.Address]map[common.Hash][]byte + copyUpdate = func(update *stateUpdate) { + accounts = append(accounts, maps.Clone(update.accounts)) + accountOrigin = append(accountOrigin, maps.Clone(update.accountsOrigin)) + storages = append(storages, maps.Clone(update.storages)) + storageOrigin = append(storageOrigin, maps.Clone(update.storagesOrigin)) } disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) @@ -250,7 +243,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[i-1] } - test.err = test.verify(root, roots[i], tdb, accountList[i], storageList[i]) + test.err = test.verify(root, roots[i], tdb, accounts[i], accountOrigin[i], storages[i], storageOrigin[i]) if test.err != nil { return false } @@ -265,7 +258,7 @@ func (test *stateTest) run() bool { // - the account was indeed not present in trie // - the account is present in new trie, nil->nil is regarded as invalid // - the slots transition is correct -func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, storages map[common.Hash][]byte, storagesOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -282,6 +275,13 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if len(nBlob) == 0 { return fmt.Errorf("missing account in new trie, %x", addrHash) } + full, err := types.FullAccountRLP(account) + if err != nil { + return err + } + if !bytes.Equal(nBlob, full) { + return fmt.Errorf("unexpected account data, want: %v, got: %v", full, nBlob) + } // Verify storage changes var nAcct types.StateAccount @@ -290,7 +290,10 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa } // Account has no slot, empty slot set is expected if nAcct.Root == types.EmptyRootHash { - if len(slots) != 0 { + if len(storagesOrigin) != 0 { + return fmt.Errorf("unexpected slot changes %x", addrHash) + } + if len(storages) != 0 { return fmt.Errorf("unexpected slot changes %x", addrHash) } return nil @@ -300,9 +303,22 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa if err != nil { return err } - for key, val := range slots { + for key, val := range storagesOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storagesOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storagesOrigin), len(storages)) + } if st.Hash() != types.EmptyRootHash { return errors.New("invalid slot changes") } @@ -316,7 +332,7 @@ func (test *stateTest) verifyAccountCreation(next common.Hash, db *triedb.Databa // - the account was indeed present in trie // - the account in old trie matches the provided value // - the slots transition is correct -func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, origin []byte, slots map[common.Hash][]byte) error { +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database, otr, ntr *trie.Trie, addr common.Address, account []byte, accountOrigin []byte, storages map[common.Hash][]byte, storageOrigin map[common.Hash][]byte) error { // Verify account change addrHash := crypto.Keccak256Hash(addr.Bytes()) oBlob, err := otr.Get(addrHash.Bytes()) @@ -330,14 +346,23 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if len(oBlob) == 0 { return fmt.Errorf("missing account in old trie, %x", addrHash) } - full, err := types.FullAccountRLP(origin) + full, err := types.FullAccountRLP(accountOrigin) if err != nil { return err } if !bytes.Equal(full, oBlob) { return fmt.Errorf("account value is not matched, %x", addrHash) } - + if len(nBlob) == 0 { + if len(account) != 0 { + return errors.New("unexpected account data") + } + } else { + full, _ = types.FullAccountRLP(account) + if !bytes.Equal(full, nBlob) { + return fmt.Errorf("unexpected account data, %x, want %v, got: %v", addrHash, full, nBlob) + } + } // Decode accounts var ( oAcct types.StateAccount @@ -361,16 +386,29 @@ func (test *stateTest) verifyAccountUpdate(next common.Hash, db *triedb.Database if err != nil { return err } - for key, val := range slots { + for key, val := range storageOrigin { + if _, exist := storages[key]; !exist { + return errors.New("storage data is not found") + } + got, err := st.Get(key.Bytes()) + if err != nil { + return err + } + if !bytes.Equal(got, storages[key]) { + return fmt.Errorf("unexpected storage data, want: %v, got: %v", storages[key], got) + } st.Update(key.Bytes(), val) } + if len(storageOrigin) != len(storages) { + return fmt.Errorf("extra storage found, want: %d, got: %d", len(storageOrigin), len(storages)) + } if st.Hash() != oAcct.Root { return errors.New("invalid slot changes") } return nil } -func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accountsOrigin map[common.Address][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { +func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Database, accounts map[common.Hash][]byte, accountsOrigin map[common.Address][]byte, storages map[common.Hash]map[common.Hash][]byte, storagesOrigin map[common.Address]map[common.Hash][]byte) error { otr, err := trie.New(trie.StateTrieID(root), db) if err != nil { return err @@ -379,12 +417,15 @@ func (test *stateTest) verify(root common.Hash, next common.Hash, db *triedb.Dat if err != nil { return err } - for addr, account := range accountsOrigin { - var err error - if len(account) == 0 { - err = test.verifyAccountCreation(next, db, otr, ntr, addr, storagesOrigin[addr]) + for addr, accountOrigin := range accountsOrigin { + var ( + err error + addrHash = crypto.Keccak256Hash(addr.Bytes()) + ) + if len(accountOrigin) == 0 { + err = test.verifyAccountCreation(next, db, otr, ntr, addr, accounts[addrHash], storages[addrHash], storagesOrigin[addr]) } else { - err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accountsOrigin[addr], storagesOrigin[addr]) + err = test.verifyAccountUpdate(next, db, otr, ntr, addr, accounts[addrHash], accountsOrigin[addr], storages[addrHash], storagesOrigin[addr]) } if err != nil { return err diff --git a/core/state/statedb_hooked_test.go b/core/state/statedb_hooked_test.go index 9abd76b02d..5f82ed06d0 100644 --- a/core/state/statedb_hooked_test.go +++ b/core/state/statedb_hooked_test.go @@ -35,7 +35,7 @@ func TestBurn(t *testing.T) { // the following occur: // 1. contract B creates contract A // 2. contract A is destructed - // 3. constract B sends ether to A + // 3. contract B sends ether to A var burned = new(uint256.Int) s, _ := New(types.EmptyRootHash, NewDatabaseForTesting()) diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 3647397df6..57886e6e03 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1305,12 +1305,12 @@ func TestDeleteStorage(t *testing.T) { obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root - _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } - _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) + _, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot) if err != nil { t.Fatal(err) } diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index c9231f0526..a320b72f11 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -17,6 +17,8 @@ package state import ( + "maps" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/trie/trienode" @@ -33,6 +35,7 @@ type contractCode struct { type accountDelete struct { address common.Address // address is the unique account identifier origin []byte // origin is the original value of account data in slim-RLP encoding. + storages map[common.Hash][]byte // storages stores mutated slots, the value should be nil. storagesOrigin map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in prefix-zero-trimmed RLP format. } @@ -52,7 +55,6 @@ type accountUpdate struct { type stateUpdate struct { originRoot common.Hash // hash of the state before applying mutation root common.Hash // hash of the state after applying mutation - destructs map[common.Hash]struct{} // destructs contains the list of destructed accounts accounts map[common.Hash][]byte // accounts stores mutated accounts in 'slim RLP' encoding accountsOrigin map[common.Address][]byte // accountsOrigin stores the original values of mutated accounts in 'slim RLP' encoding storages map[common.Hash]map[common.Hash][]byte // storages stores mutated slots in 'prefix-zero-trimmed' RLP format @@ -71,7 +73,6 @@ func (sc *stateUpdate) empty() bool { // account deletions and account updates to form a comprehensive state update. func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common.Hash]*accountDelete, updates map[common.Hash]*accountUpdate, nodes *trienode.MergedNodeSet) *stateUpdate { var ( - destructs = make(map[common.Hash]struct{}) accounts = make(map[common.Hash][]byte) accountsOrigin = make(map[common.Address][]byte) storages = make(map[common.Hash]map[common.Hash][]byte) @@ -82,8 +83,12 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // within the same block, the deletions must be aggregated first. for addrHash, op := range deletes { addr := op.address - destructs[addrHash] = struct{}{} + accounts[addrHash] = nil accountsOrigin[addr] = op.origin + + if len(op.storages) > 0 { + storages[addrHash] = op.storages + } if len(op.storagesOrigin) > 0 { storagesOrigin[addr] = op.storagesOrigin } @@ -95,35 +100,41 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common if op.code != nil { codes[addr] = *op.code } - // Aggregate the account changes. The original account value will only - // be tracked if it's not present yet. accounts[addrHash] = op.data + + // Aggregate the account original value. If the account is already + // present in the aggregated accountsOrigin set, skip it. if _, found := accountsOrigin[addr]; !found { accountsOrigin[addr] = op.origin } - // Aggregate the storage changes. The original storage slot value will - // only be tracked if it's not present yet. + // Aggregate the storage mutation list. If a slot in op.storages is + // already present in aggregated storages set, the value will be + // overwritten. if len(op.storages) > 0 { - storages[addrHash] = op.storages - } - if len(op.storagesOrigin) > 0 { - origin := storagesOrigin[addr] - if origin == nil { - storagesOrigin[addr] = op.storagesOrigin - continue + if _, exist := storages[addrHash]; !exist { + storages[addrHash] = op.storages + } else { + maps.Copy(storages[addrHash], op.storages) } - for key, slot := range op.storagesOrigin { - if _, found := origin[key]; !found { - origin[key] = slot + } + // Aggregate the storage original values. If the slot is already present + // in aggregated storagesOrigin set, skip it. + if len(op.storagesOrigin) > 0 { + origin, exist := storagesOrigin[addr] + if !exist { + storagesOrigin[addr] = op.storagesOrigin + } else { + for key, slot := range op.storagesOrigin { + if _, found := origin[key]; !found { + origin[key] = slot + } } } - storagesOrigin[addr] = origin } } return &stateUpdate{ originRoot: types.TrieRootHash(originRoot), root: types.TrieRootHash(root), - destructs: destructs, accounts: accounts, accountsOrigin: accountsOrigin, storages: storages, @@ -139,7 +150,6 @@ func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common // package. func (sc *stateUpdate) stateSet() *triedb.StateSet { return &triedb.StateSet{ - Destructs: sc.destructs, Accounts: sc.accounts, AccountsOrigin: sc.accountsOrigin, Storages: sc.storages, diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 31405fa078..3b987bb289 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -49,7 +49,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c header = block.Header() gaspool = new(GasPool).AddGas(block.GasLimit()) blockContext = NewEVMBlockContext(header, p.chain, nil) - evm = vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) + evm = vm.NewEVM(blockContext, statedb, p.config, cfg) signer = types.MakeSigner(p.config, header.Number, header.Time) ) // Iterate over and process the individual transactions @@ -65,7 +65,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c return // Also invalid block, bail out } statedb.SetTxContext(tx.Hash(), i) - if err := precacheTransaction(msg, p.config, gaspool, statedb, header, evm); err != nil { + if err := precacheTransaction(msg, gaspool, evm); err != nil { return // Ugh, something went horribly wrong, bail out } // If we're pre-byzantium, pre-load trie nodes for the intermediate root @@ -82,9 +82,9 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // precacheTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. -func precacheTransaction(msg *Message, config *params.ChainConfig, gaspool *GasPool, statedb *state.StateDB, header *types.Header, evm *vm.EVM) error { +func precacheTransaction(msg *Message, gaspool *GasPool, evm *vm.EVM) error { // Update the evm with the new transaction context. - evm.Reset(NewEVMTxContext(msg), statedb) + evm.SetTxContext(NewEVMTxContext(msg)) // Add addresses to access list if applicable _, err := ApplyMessage(evm, msg, gaspool) return err diff --git a/core/state_processor.go b/core/state_processor.go index c04049e986..1703377111 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -74,18 +74,18 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg ) // Apply pre-execution system calls. - context = NewEVMBlockContext(header, p.chain, nil) - - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) var tracingStateDB = vm.StateDB(statedb) if hooks := cfg.Tracer; hooks != nil { tracingStateDB = state.NewHookedState(statedb, hooks) } + context = NewEVMBlockContext(header, p.chain, nil) + evm := vm.NewEVM(context, tracingStateDB, p.config, cfg) + if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - ProcessBeaconBlockRoot(*beaconRoot, vmenv, tracingStateDB) + ProcessBeaconBlockRoot(*beaconRoot, evm) } if p.config.IsPrague(block.Number(), block.Time()) { - ProcessParentBlockHash(block.ParentHash(), vmenv, tracingStateDB) + ProcessParentBlockHash(block.ParentHash(), evm) } // Iterate over and process the individual transactions @@ -96,7 +96,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } statedb.SetTxContext(tx.Hash(), i) - receipt, err := ApplyTransactionWithEVM(msg, p.config, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) + receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, tx, usedGas, evm) if err != nil { return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -113,10 +113,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg } requests = append(requests, depositRequests) // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(vmenv, tracingStateDB) + withdrawalRequests := ProcessWithdrawalQueue(evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(vmenv, tracingStateDB) + consolidationRequests := ProcessConsolidationQueue(evm) requests = append(requests, consolidationRequests) } @@ -134,10 +134,8 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { - var tracingStateDB = vm.StateDB(statedb) +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { if hooks := evm.Config.Tracer; hooks != nil { - tracingStateDB = state.NewHookedState(statedb, hooks) if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) } @@ -148,7 +146,7 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) - evm.Reset(txContext, tracingStateDB) + evm.SetTxContext(txContext) // Apply the transaction to the current state (included in the env). result, err := ApplyMessage(evm, msg, gp) @@ -158,10 +156,10 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo // Update the state with pending changes. var root []byte - if config.IsByzantium(blockNumber) { - tracingStateDB.Finalise(true) + if evm.ChainConfig().IsByzantium(blockNumber) { + evm.StateDB.Finalise(true) } else { - root = statedb.IntermediateRoot(config.IsEIP158(blockNumber)).Bytes() + root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } *usedGas += result.UsedGas @@ -210,22 +208,19 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // and uses the input parameters for its environment. It returns the receipt // for the transaction, gas used and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64, cfg vm.Config) (*types.Receipt, error) { - msg, err := TransactionToMessage(tx, types.MakeSigner(config, header.Number, header.Time), header.BaseFee) +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { + msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - blockContext := NewEVMBlockContext(header, bc, author) - txContext := NewEVMTxContext(msg) - vmenv := vm.NewEVM(blockContext, txContext, statedb, config, cfg) - return ApplyTransactionWithEVM(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), tx, usedGas, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root // contract. This method is exported to be used in tests. -func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -242,16 +237,16 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb vm.St To: ¶ms.BeaconRootsAddress, Data: beaconRoot[:], } - vmenv.Reset(NewEVMTxContext(msg), statedb) - statedb.AddAddressToAccessList(params.BeaconRootsAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.BeaconRootsAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessParentBlockHash stores the parent block hash in the history storage contract // as per EIP-2935. -func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.StateDB) { - if tracer := vmenv.Config.Tracer; tracer != nil { +func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -268,26 +263,26 @@ func ProcessParentBlockHash(prevHash common.Hash, vmenv *vm.EVM, statedb vm.Stat To: ¶ms.HistoryStorageAddress, Data: prevHash.Bytes(), } - vmenv.Reset(NewEVMTxContext(msg), statedb) - statedb.AddAddressToAccessList(params.HistoryStorageAddress) - _, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(params.HistoryStorageAddress) + _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) } // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(vmenv *vm.EVM, statedb vm.StateDB) []byte { - return processRequestsSystemCall(vmenv, statedb, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(evm *vm.EVM) []byte { + return processRequestsSystemCall(evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType byte, addr common.Address) []byte { - if tracer := vmenv.Config.Tracer; tracer != nil { +func processRequestsSystemCall(evm *vm.EVM, requestType byte, addr common.Address) []byte { + if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() } @@ -295,7 +290,6 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by defer tracer.OnSystemCallEnd() } } - msg := &Message{ From: params.SystemAddress, GasLimit: 30_000_000, @@ -304,10 +298,10 @@ func processRequestsSystemCall(vmenv *vm.EVM, statedb vm.StateDB, requestType by GasTipCap: common.Big0, To: &addr, } - vmenv.Reset(NewEVMTxContext(msg), statedb) - statedb.AddAddressToAccessList(addr) - ret, _, _ := vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) - statedb.Finalise(true) + evm.SetTxContext(NewEVMTxContext(msg)) + evm.StateDB.AddAddressToAccessList(addr) + ret, _, _ := evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + evm.StateDB.Finalise(true) // Create withdrawals requestsData with prefix 0x01 requestsData := make([]byte, len(ret)+1) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 0352ea9783..02d339f99c 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1714,3 +1714,53 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { } return txpool.TxStatusUnknown } + +// Clear implements txpool.SubPool, removing all tracked transactions +// from the blob pool and persistent store. +func (p *BlobPool) Clear() { + p.lock.Lock() + defer p.lock.Unlock() + + // manually iterating and deleting every entry is super sub-optimal + // However, Clear is not currently used in production so + // performance is not critical at the moment. + for hash := range p.lookup.txIndex { + id, _ := p.lookup.storeidOfTx(hash) + if err := p.store.Delete(id); err != nil { + log.Warn("failed to delete blob tx from backing store", "err", err) + } + } + for hash := range p.lookup.blobIndex { + id, _ := p.lookup.storeidOfBlob(hash) + if err := p.store.Delete(id); err != nil { + log.Warn("failed to delete blob from backing store", "err", err) + } + } + + // unreserve each tracked account. Ideally, we could just clear the + // reservation map in the parent txpool context. However, if we clear in + // parent context, to avoid exposing the subpool lock, we have to lock the + // reservations and then lock each subpool. + // + // This creates the potential for a deadlock situation: + // + // * TxPool.Clear locks the reservations + // * a new transaction is received which locks the subpool mutex + // * TxPool.Clear attempts to lock subpool mutex + // + // The transaction addition may attempt to reserve the sender addr which + // can't happen until Clear releases the reservation lock. Clear cannot + // acquire the subpool lock until the transaction addition is completed. + for acct, _ := range p.index { + p.reserve(acct, false) + } + p.lookup = newLookup() + p.index = make(map[common.Address][]*blobTxMeta) + p.spent = make(map[common.Address]*uint256.Int) + + var ( + basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head)) + blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) + ) + p.evict = newPriceHeap(basefee, blobfee, p.index) +} diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index f7495dd39f..89ff86df02 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1961,3 +1961,44 @@ func (t *lookup) RemotesBelowTip(threshold *big.Int) types.Transactions { func numSlots(tx *types.Transaction) int { return int((tx.Size() + txSlotSize - 1) / txSlotSize) } + +// Clear implements txpool.SubPool, removing all tracked txs from the pool +// and rotating the journal. +func (pool *LegacyPool) Clear() { + pool.mu.Lock() + defer pool.mu.Unlock() + + // unreserve each tracked account. Ideally, we could just clear the + // reservation map in the parent txpool context. However, if we clear in + // parent context, to avoid exposing the subpool lock, we have to lock the + // reservations and then lock each subpool. + // + // This creates the potential for a deadlock situation: + // + // * TxPool.Clear locks the reservations + // * a new transaction is received which locks the subpool mutex + // * TxPool.Clear attempts to lock subpool mutex + // + // The transaction addition may attempt to reserve the sender addr which + // can't happen until Clear releases the reservation lock. Clear cannot + // acquire the subpool lock until the transaction addition is completed. + for _, tx := range pool.all.remotes { + senderAddr, _ := types.Sender(pool.signer, tx) + pool.reserve(senderAddr, false) + } + for localSender, _ := range pool.locals.accounts { + pool.reserve(localSender, false) + } + + pool.all = newLookup() + pool.priced = newPricedList(pool.all) + pool.pending = make(map[common.Address]*list) + pool.queue = make(map[common.Address]*list) + + if !pool.config.NoLocals && pool.config.Journal != "" { + pool.journal = newTxJournal(pool.config.Journal) + if err := pool.journal.rotate(pool.local()); err != nil { + log.Warn("Failed to rotate transaction journal", "err", err) + } + } +} diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 180facd217..9ee0a69c0b 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -168,4 +168,7 @@ type SubPool interface { // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. Status(hash common.Hash) TxStatus + + // Clear removes all tracked transactions from the pool + Clear() } diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 54ae3be569..ce455e806e 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -497,3 +497,10 @@ func (p *TxPool) Sync() error { return errors.New("pool already terminated") } } + +// Clear removes all tracked txs from the subpools. +func (p *TxPool) Clear() { + for _, subpool := range p.subpools { + subpool.Clear() + } +} diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 5a4210cdab..45b317d3c0 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -225,8 +225,8 @@ func TestProcessParentBlockHash(t *testing.T) { for i := 1; i <= num; i++ { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, params.MergedTestChainConfig, vm.Config{}) - ProcessParentBlockHash(header.ParentHash, evm, statedb) + evm := vm.NewEVM(vmContext, statedb, params.MergedTestChainConfig, vm.Config{}) + ProcessParentBlockHash(header.ParentHash, evm) } // Read block hashes for block 0 .. num-1 for i := 0; i < num; i++ { @@ -338,7 +338,7 @@ func TestProcessVerkleInvalidContractCreation(t *testing.T) { } } } else if bytes.Equal(stemStateDiff.Stem[:], tx1ContractStem) { - // For this contract creation, check that only the accound header and storage slot 41 + // For this contract creation, check that only the account header and storage slot 41 // are found in the witness. for _, suffixDiff := range stemStateDiff.SuffixDiffs { if suffixDiff.Suffix != 105 && suffixDiff.Suffix != 0 && suffixDiff.Suffix != 1 { @@ -1033,7 +1033,7 @@ func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount ) // Prefund the account, at an address that the contract will be deployed at, // before it selfdestrucs. We can therefore check that the account itseld is - // NOT destroyed, which is what the currrent version of the spec requires. + // NOT destroyed, which is what the current version of the spec requires. // TODO(gballet) revisit after the spec has been modified. gspec.Alloc[contract] = types.Account{ Balance: big.NewInt(100), diff --git a/core/vm/eof.go b/core/vm/eof.go index fcced6ae75..a5406283d5 100644 --- a/core/vm/eof.go +++ b/core/vm/eof.go @@ -96,7 +96,7 @@ func (meta *functionMetadata) checkInputs(stackMin int) error { } // checkStackMax checks the if current maximum stack combined with the -// functin max stack will result in a stack overflow, and if so returns an error. +// function max stack will result in a stack overflow, and if so returns an error. func (meta *functionMetadata) checkStackMax(stackMax int) error { newMaxStack := stackMax + int(meta.maxStackHeight) - int(meta.inputs) if newMaxStack > int(params.StackLimit) { diff --git a/core/vm/evm.go b/core/vm/evm.go index 0593a32a3e..34e5fa766b 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -116,12 +116,13 @@ type EVM struct { precompiles map[common.Address]PrecompiledContract } -// NewEVM returns a new EVM. The returned EVM is not thread safe and should -// only ever be used *once*. -func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig *params.ChainConfig, config Config) *EVM { +// NewEVM constructs an EVM instance with the supplied block context, state +// database and several configs. It meant to be used throughout the entire +// state transition of a block, with the transaction context switched as +// needed by calling evm.SetTxContext. +func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainConfig, config Config) *EVM { evm := &EVM{ Context: blockCtx, - TxContext: txCtx, StateDB: statedb, Config: config, chainConfig: chainConfig, @@ -132,6 +133,11 @@ func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig return evm } +// SetTracer sets the tracer for following state transition. +func (evm *EVM) SetTracer(tracer *tracing.Hooks) { + evm.Config.Tracer = tracer +} + // SetPrecompiles sets the precompiled contracts for the EVM. // This method is only used through RPC calls. // It is not thread-safe. @@ -139,14 +145,13 @@ func (evm *EVM) SetPrecompiles(precompiles PrecompiledContracts) { evm.precompiles = precompiles } -// Reset resets the EVM with a new transaction context.Reset +// SetTxContext resets the EVM with a new transaction context. // This is not threadsafe and should only be done very cautiously. -func (evm *EVM) Reset(txCtx TxContext, statedb StateDB) { +func (evm *EVM) SetTxContext(txCtx TxContext) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(statedb.PointCache()) + txCtx.AccessEvents = state.NewAccessEvents(evm.StateDB.PointCache()) } evm.TxContext = txCtx - evm.StateDB = statedb } // Cancel cancels any running EVM operation. This may be called concurrently and diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index babe9a5b6a..be86885261 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -95,16 +95,16 @@ func TestEIP2200(t *testing.T) { CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true }, Transfer: func(StateDB, common.Address, common.Address, *uint256.Int) {}, } - vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, Config{ExtraEips: []int{2200}}) - _, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) + _, gas, err := evm.Call(AccountRef(common.Address{}), address, nil, tt.gaspool, new(uint256.Int)) if !errors.Is(err, tt.failure) { t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure) } if used := tt.gaspool - gas; used != tt.used { t.Errorf("test %d: gas used mismatch: have %v, want %v", i, used, tt.used) } - if refund := vmenv.StateDB.GetRefund(); refund != tt.refund { + if refund := evm.StateDB.GetRefund(); refund != tt.refund { t.Errorf("test %d: gas refund mismatch: have %v, want %v", i, refund, tt.refund) } } @@ -151,9 +151,9 @@ func TestCreateGas(t *testing.T) { config.ExtraEips = []int{3860} } - vmenv := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, config) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, config) var startGas = uint64(testGas) - ret, gas, err := vmenv.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) + ret, gas, err := evm.Call(AccountRef(common.Address{}), address, nil, startGas, new(uint256.Int)) if err != nil { return false } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index b8e62e1de5..08f2b2bfea 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -104,10 +104,9 @@ func init() { func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) for i, test := range tests { @@ -116,7 +115,7 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu expected := new(uint256.Int).SetBytes(common.Hex2Bytes(test.Expected)) stack.push(x) stack.push(y) - opFn(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opFn(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data)) } @@ -203,10 +202,9 @@ func TestSAR(t *testing.T) { func TestAddMod(t *testing.T) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - evmInterpreter = NewEVMInterpreter(env) - pc = uint64(0) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) tests := []struct { x string @@ -231,7 +229,7 @@ func TestAddMod(t *testing.T) { stack.push(z) stack.push(y) stack.push(x) - opAddmod(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opAddmod(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() if actual.Cmp(expected) != 0 { t.Errorf("Testcase %d, expected %x, got %x", i, expected, actual) @@ -247,10 +245,9 @@ func TestWriteExpectedValues(t *testing.T) { // getResult is a convenience function to generate the expected values getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) result := make([]TwoOperandTestcase, len(args)) for i, param := range args { @@ -258,7 +255,7 @@ func TestWriteExpectedValues(t *testing.T) { y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) stack.push(x) stack.push(y) - opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) + opFn(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) actual := stack.pop() result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} } @@ -292,13 +289,10 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - scope = &ScopeContext{nil, stack, nil} - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + scope = &ScopeContext{nil, stack, nil} ) - - env.interpreter = evmInterpreter // convert args intArgs := make([]*uint256.Int, len(args)) for i, arg := range args { @@ -310,7 +304,7 @@ func opBenchmark(bench *testing.B, op executionFunc, args ...string) { for _, arg := range intArgs { stack.push(arg) } - op(&pc, evmInterpreter, scope) + op(&pc, evm.interpreter, scope) stack.pop() } bench.StopTimer() @@ -533,25 +527,22 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - - env.interpreter = evmInterpreter mem.Resize(64) pc := uint64(0) v := "abcdef00000000000000abba000000000deaf000000c0de00100000000133700" stack.push(new(uint256.Int).SetBytes(common.Hex2Bytes(v))) stack.push(new(uint256.Int)) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) if got := common.Bytes2Hex(mem.GetCopy(0, 32)); got != v { t.Fatalf("Mstore fail, got %v, expected %v", got, v) } stack.push(new(uint256.Int).SetUint64(0x1)) stack.push(new(uint256.Int)) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) if common.Bytes2Hex(mem.GetCopy(0, 32)) != "0000000000000000000000000000000000000000000000000000000000000001" { t.Fatalf("Mstore failed to overwrite previous value") } @@ -559,13 +550,10 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - - env.interpreter = evmInterpreter mem.Resize(64) pc := uint64(0) memStart := new(uint256.Int) @@ -575,43 +563,41 @@ func BenchmarkOpMstore(bench *testing.B) { for i := 0; i < bench.N; i++ { stack.push(value) stack.push(memStart) - opMstore(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMstore(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) } } func TestOpTstore(t *testing.T) { var ( - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - env = NewEVM(BlockContext{}, TxContext{}, statedb, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) - caller = common.Address{} - to = common.Address{1} - contractRef = contractRef{caller} - contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) - scopeContext = ScopeContext{mem, stack, contract} - value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + evm = NewEVM(BlockContext{}, statedb, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() + caller = common.Address{} + to = common.Address{1} + contractRef = contractRef{caller} + contract = NewContract(contractRef, AccountRef(to), new(uint256.Int), 0) + scopeContext = ScopeContext{mem, stack, contract} + value = common.Hex2Bytes("abcdef00000000000000abba000000000deaf000000c0de00100000000133700") ) // Add a stateObject for the caller and the contract being called statedb.CreateAccount(caller) statedb.CreateAccount(to) - env.interpreter = evmInterpreter pc := uint64(0) // push the value to the stack stack.push(new(uint256.Int).SetBytes(value)) // push the location to the stack stack.push(new(uint256.Int)) - opTstore(&pc, evmInterpreter, &scopeContext) + opTstore(&pc, evm.interpreter, &scopeContext) // there should be no elements on the stack after TSTORE if stack.len() != 0 { t.Fatal("stack wrong size") } // push the location to the stack stack.push(new(uint256.Int)) - opTload(&pc, evmInterpreter, &scopeContext) + opTload(&pc, evm.interpreter, &scopeContext) // there should be one element on the stack after TLOAD if stack.len() != 1 { t.Fatal("stack wrong size") @@ -624,12 +610,10 @@ func TestOpTstore(t *testing.T) { func BenchmarkOpKeccak256(bench *testing.B) { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - mem = NewMemory() - evmInterpreter = NewEVMInterpreter(env) + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + mem = NewMemory() ) - env.interpreter = evmInterpreter mem.Resize(32) pc := uint64(0) start := new(uint256.Int) @@ -638,7 +622,7 @@ func BenchmarkOpKeccak256(bench *testing.B) { for i := 0; i < bench.N; i++ { stack.push(uint256.NewInt(32)) stack.push(start) - opKeccak256(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opKeccak256(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) } } @@ -728,12 +712,11 @@ func TestRandom(t *testing.T) { {name: "hash(0x010203)", random: crypto.Keccak256Hash([]byte{0x01, 0x02, 0x03})}, } { var ( - env = NewEVM(BlockContext{Random: &tt.random}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{Random: &tt.random}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) - opRandom(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opRandom(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) } @@ -769,13 +752,13 @@ func TestBlobHash(t *testing.T) { {name: "out-of-bounds (nil)", idx: 25, expect: zero, hashes: nil}, } { var ( - env = NewEVM(BlockContext{}, TxContext{BlobHashes: tt.hashes}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) + evm.SetTxContext(TxContext{BlobHashes: tt.hashes}) stack.push(uint256.NewInt(tt.idx)) - opBlobHash(&pc, evmInterpreter, &ScopeContext{nil, stack, nil}) + opBlobHash(&pc, evm.interpreter, &ScopeContext{nil, stack, nil}) if len(stack.data) != 1 { t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) } @@ -872,10 +855,9 @@ func TestOpMCopy(t *testing.T) { }, } { var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - evmInterpreter = env.interpreter + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) ) data := common.FromHex(strings.ReplaceAll(tc.pre, " ", "")) // Set pre @@ -906,7 +888,7 @@ func TestOpMCopy(t *testing.T) { } // and the dynamic cost var haveGas uint64 - if dynamicCost, err := gasMcopy(env, nil, stack, mem, memorySize); err != nil { + if dynamicCost, err := gasMcopy(evm, nil, stack, mem, memorySize); err != nil { t.Error(err) } else { haveGas = GasFastestStep + dynamicCost @@ -916,7 +898,7 @@ func TestOpMCopy(t *testing.T) { mem.Resize(memorySize) } // Do the copy - opMcopy(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + opMcopy(&pc, evm.interpreter, &ScopeContext{mem, stack, nil}) want := common.FromHex(strings.ReplaceAll(tc.want, " ", "")) if have := mem.store; !bytes.Equal(want, have) { t.Errorf("case %d: \nwant: %#x\nhave: %#x\n", i, want, have) diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index 7d4b2ddf36..cacad8f813 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -47,7 +47,7 @@ func TestLoopInterrupt(t *testing.T) { statedb.SetCode(address, common.Hex2Bytes(tt)) statedb.Finalise(true) - evm := NewEVM(vmctx, TxContext{}, statedb, params.AllEthashProtocolChanges, Config{}) + evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, Config{}) errChannel := make(chan error) timeout := make(chan bool) diff --git a/core/vm/program/program.go b/core/vm/program/program.go new file mode 100644 index 0000000000..acc7fd25fc --- /dev/null +++ b/core/vm/program/program.go @@ -0,0 +1,430 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the goevmlab library. If not, see . + +// package program is a utility to create EVM bytecode for testing, but _not_ for production. As such: +// +// - There are not package guarantees. We might iterate heavily on this package, and do backwards-incompatible changes without warning +// - There are no quality-guarantees. These utilities may produce evm-code that is non-functional. YMMV. +// - There are no stability-guarantees. The utility will `panic` if the inputs do not align / make sense. +package program + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" +) + +// Program is a simple bytecode container. It can be used to construct +// simple EVM programs. Errors during construction of a Program typically +// cause panics: so avoid using these programs in production settings or on +// untrusted input. +// This package is mainly meant to aid in testing. This is not a production +// -level "compiler". +type Program struct { + code []byte +} + +// New creates a new Program +func New() *Program { + return &Program{ + code: make([]byte, 0), + } +} + +// add adds the op to the code. +func (p *Program) add(op byte) *Program { + p.code = append(p.code, op) + return p +} + +// pushBig creates a PUSHX instruction and pushes the given val. +// - If the val is nil, it pushes zero +// - If the val is bigger than 32 bytes, it panics +func (p *Program) doPush(val *uint256.Int) { + if val == nil { + val = new(uint256.Int) + } + valBytes := val.Bytes() + if len(valBytes) == 0 { + valBytes = append(valBytes, 0) + } + bLen := len(valBytes) + p.add(byte(vm.PUSH1) - 1 + byte(bLen)) + p.Append(valBytes) +} + +// Append appends the given data to the code. +func (p *Program) Append(data []byte) *Program { + p.code = append(p.code, data...) + return p +} + +// Bytes returns the Program bytecode. OBS: This is not a copy. +func (p *Program) Bytes() []byte { + return p.code +} + +// SetBytes sets the Program bytecode. The combination of Bytes and SetBytes means +// that external callers can implement missing functionality: +// +// ... +// prog.Push(1) +// code := prog.Bytes() +// manipulate(code) +// prog.SetBytes(code) +func (p *Program) SetBytes(code []byte) { + p.code = code +} + +// Hex returns the Program bytecode as a hex string. +func (p *Program) Hex() string { + return fmt.Sprintf("%02x", p.Bytes()) +} + +// Op appends the given opcode(s). +func (p *Program) Op(ops ...vm.OpCode) *Program { + for _, op := range ops { + p.add(byte(op)) + } + return p +} + +// Push creates a PUSHX instruction with the data provided. If zero is being pushed, +// PUSH0 will be avoided in favour of [PUSH1 0], to ensure backwards compatibility. +func (p *Program) Push(val any) *Program { + switch v := val.(type) { + case int: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case uint64: + p.doPush(new(uint256.Int).SetUint64(v)) + case uint32: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case uint16: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case *big.Int: + p.doPush(uint256.MustFromBig(v)) + case *uint256.Int: + p.doPush(v) + case uint256.Int: + p.doPush(&v) + case []byte: + p.doPush(new(uint256.Int).SetBytes(v)) + case byte: + p.doPush(new(uint256.Int).SetUint64(uint64(v))) + case interface{ Bytes() []byte }: + // Here, we jump through some hoops in order to avoid depending on + // go-ethereum types.Address and common.Hash, and instead use the + // interface. This works on both values and pointers! + p.doPush(new(uint256.Int).SetBytes(v.Bytes())) + case nil: + p.doPush(nil) + default: + panic(fmt.Sprintf("unsupported type %T", v)) + } + return p +} + +// Push0 implements PUSH0 (0x5f). +func (p *Program) Push0() *Program { + return p.Op(vm.PUSH0) +} + +// ExtcodeCopy performs an extcodecopy invocation. +func (p *Program) ExtcodeCopy(address, memOffset, codeOffset, length any) *Program { + p.Push(length) + p.Push(codeOffset) + p.Push(memOffset) + p.Push(address) + return p.Op(vm.EXTCODECOPY) +} + +// Call is a convenience function to make a call. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) Call(gas *uint256.Int, address, value, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize && value == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset).Push(value) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.CALL) +} + +// DelegateCall is a convenience function to make a delegatecall. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) DelegateCall(gas *uint256.Int, address, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.DELEGATECALL) +} + +// StaticCall is a convenience function to make a staticcall. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) StaticCall(gas *uint256.Int, address, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.STATICCALL) +} + +// StaticCall is a convenience function to make a callcode. If 'gas' is nil, the opcode GAS will +// be used to provide all gas. +func (p *Program) CallCode(gas *uint256.Int, address, value, inOffset, inSize, outOffset, outSize any) *Program { + if outOffset == outSize && inSize == outSize && inOffset == outSize { + p.Push(outSize).Op(vm.DUP1, vm.DUP1, vm.DUP1) + } else { + p.Push(outSize).Push(outOffset).Push(inSize).Push(inOffset) + } + p.Push(value) + p.Push(address) + if gas == nil { + p.Op(vm.GAS) + } else { + p.doPush(gas) + } + return p.Op(vm.CALLCODE) +} + +// Label returns the PC (of the next instruction). +func (p *Program) Label() uint64 { + return uint64(len(p.code)) +} + +// Jumpdest adds a JUMPDEST op, and returns the PC of that instruction. +func (p *Program) Jumpdest() (*Program, uint64) { + here := p.Label() + p.Op(vm.JUMPDEST) + return p, here +} + +// Jump pushes the destination and adds a JUMP. +func (p *Program) Jump(loc any) *Program { + p.Push(loc) + p.Op(vm.JUMP) + return p +} + +// JumpIf implements JUMPI. +func (p *Program) JumpIf(loc any, condition any) *Program { + p.Push(condition) + p.Push(loc) + p.Op(vm.JUMPI) + return p +} + +// Size returns the current size of the bytecode. +func (p *Program) Size() int { + return len(p.code) +} + +// InputAddressToStack stores the input (calldata) to memory as address (20 bytes). +func (p *Program) InputAddressToStack(inputOffset uint32) *Program { + p.Push(inputOffset) + p.Op(vm.CALLDATALOAD) // Loads [n -> n + 32] of input data to stack top + mask, _ := big.NewInt(0).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 16) + p.Push(mask) // turn into address + return p.Op(vm.AND) +} + +// MStore stores the provided data (into the memory area starting at memStart). +func (p *Program) Mstore(data []byte, memStart uint32) *Program { + var idx = 0 + // We need to store it in chunks of 32 bytes + for ; idx+32 <= len(data); idx += 32 { + chunk := data[idx : idx+32] + // push the value + p.Push(chunk) + // push the memory index + p.Push(uint32(idx) + memStart) + p.Op(vm.MSTORE) + } + // Remainders become stored using MSTORE8 + for ; idx < len(data); idx++ { + b := data[idx] + // push the byte + p.Push(b) + p.Push(uint32(idx) + memStart) + p.Op(vm.MSTORE8) + } + return p +} + +// MstoreSmall stores the provided data, which must be smaller than 32 bytes, +// into the memory area starting at memStart. +// The data will be LHS zero-added to align on 32 bytes. +// For example, providing data 0x1122, it will do a PUSH2: +// PUSH2 0x1122, resulting in +// stack: 0x0000000000000000000000000000000000000000000000000000000000001122 +// followed by MSTORE(0,0) +// And thus, the resulting memory will be +// [ 0000000000000000000000000000000000000000000000000000000000001122 ] +func (p *Program) MstoreSmall(data []byte, memStart uint32) *Program { + if len(data) > 32 { + // For larger sizes, use Mstore instead. + panic("only <=32 byte data size supported") + } + if len(data) == 0 { + // Storing 0-length data smells of an error somewhere. + panic("data is zero length") + } + // push the value + p.Push(data) + // push the memory index + p.Push(memStart) + p.Op(vm.MSTORE) + return p +} + +// MemToStorage copies the given memory area into SSTORE slots, +// It expects data to be aligned to 32 byte, and does not zero out +// remainders if some data is not +// I.e, if given a 1-byte area, it will still copy the full 32 bytes to storage. +func (p *Program) MemToStorage(memStart, memSize, startSlot int) *Program { + // We need to store it in chunks of 32 bytes + for idx := memStart; idx < (memStart + memSize); idx += 32 { + dataStart := idx + // Mload the chunk + p.Push(dataStart) + p.Op(vm.MLOAD) + // Value is now on stack, + p.Push(startSlot) + p.Op(vm.SSTORE) + startSlot++ + } + return p +} + +// ReturnViaCodeCopy utilises CODECOPY to place the given data in the bytecode of +// p, loads into memory (offset 0) and returns the code. +// This is a typical "constructor". +// Note: since all indexing is calculated immediately, the preceding bytecode +// must not be expanded or shortened. +func (p *Program) ReturnViaCodeCopy(data []byte) *Program { + p.Push(len(data)) + // For convenience, we'll use PUSH2 for the offset. Then we know we can always + // fit, since code is limited to 0xc000 + p.Op(vm.PUSH2) + offsetPos := p.Size() // Need to update this position later on + p.Append([]byte{0, 0}) // Offset of the code to be copied + p.Push(0) // Offset in memory (destination) + p.Op(vm.CODECOPY) // Copy from code[offset:offset+len] to memory[0:] + p.Return(0, len(data)) // Return memory[0:len] + offset := p.Size() + p.Append(data) // And add the data + + // Now, go back and fix the offset + p.code[offsetPos] = byte(offset >> 8) + p.code[offsetPos+1] = byte(offset) + return p +} + +// Sstore stores the given byte array to the given slot. +// OBS! Does not verify that the value indeed fits into 32 bytes. +// If it does not, it will panic later on via doPush. +func (p *Program) Sstore(slot any, value any) *Program { + p.Push(value) + p.Push(slot) + return p.Op(vm.SSTORE) +} + +// Tstore stores the given byte array to the given t-slot. +// OBS! Does not verify that the value indeed fits into 32 bytes. +// If it does not, it will panic later on via doPush. +func (p *Program) Tstore(slot any, value any) *Program { + p.Push(value) + p.Push(slot) + return p.Op(vm.TSTORE) +} + +// Return implements RETURN +func (p *Program) Return(offset, len int) *Program { + p.Push(len) + p.Push(offset) + return p.Op(vm.RETURN) +} + +// ReturnData loads the given data into memory, and does a return with it +func (p *Program) ReturnData(data []byte) *Program { + p.Mstore(data, 0) + return p.Return(0, len(data)) +} + +// Create2 uses create2 to construct a contract with the given bytecode. +// This operation leaves either '0' or address on the stack. +func (p *Program) Create2(code []byte, salt any) *Program { + var ( + value = 0 + offset = 0 + size = len(code) + ) + // Load the code into mem + p.Mstore(code, 0) + // Create it + return p.Push(salt). + Push(size). + Push(offset). + Push(value). + Op(vm.CREATE2) + // On the stack now, is either + // - zero: in case of failure, OR + // - address: in case of success +} + +// Create2ThenCall calls create2 with the given initcode and salt, and then calls +// into the created contract (or calls into zero, if the creation failed). +func (p *Program) Create2ThenCall(code []byte, salt any) *Program { + p.Create2(code, salt) + // If there happen to be a zero on the stack, it doesn't matter, we're + // not sending any value anyway + p.Push(0).Push(0) // mem out + p.Push(0).Push(0) // mem in + p.Push(0) // value + p.Op(vm.DUP6) // address + p.Op(vm.GAS) + p.Op(vm.CALL) + p.Op(vm.POP) // pop the retval + return p.Op(vm.POP) // pop the address +} + +// Selfdestruct pushes beneficiary and invokes selfdestruct. +func (p *Program) Selfdestruct(beneficiary any) *Program { + p.Push(beneficiary) + return p.Op(vm.SELFDESTRUCT) +} diff --git a/core/vm/program/program_test.go b/core/vm/program/program_test.go new file mode 100644 index 0000000000..0b34210067 --- /dev/null +++ b/core/vm/program/program_test.go @@ -0,0 +1,311 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the goevmlab library. If not, see . + +package program + +import ( + "bytes" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/holiman/uint256" +) + +func TestPush(t *testing.T) { + tests := []struct { + input interface{} + expected string + }{ + // native ints + {0, "6000"}, + {0xfff, "610fff"}, + {nil, "6000"}, + {uint8(1), "6001"}, + {uint16(1), "6001"}, + {uint32(1), "6001"}, + {uint64(1), "6001"}, + // bigints + {big.NewInt(0), "6000"}, + {big.NewInt(1), "6001"}, + {big.NewInt(0xfff), "610fff"}, + // uint256 + {uint256.NewInt(1), "6001"}, + {uint256.Int{1, 0, 0, 0}, "6001"}, + // Addresses + {common.HexToAddress("0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"), "73deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"}, + {&common.Address{}, "6000"}, + } + for i, tc := range tests { + have := New().Push(tc.input).Hex() + if have != tc.expected { + t.Errorf("test %d: got %v expected %v", i, have, tc.expected) + } + } +} + +func TestCall(t *testing.T) { + { // Nil gas + have := New().Call(nil, common.HexToAddress("0x1337"), big.NewInt(1), 1, 2, 3, 4).Hex() + want := "600460036002600160016113375af1" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // Non nil gas + have := New().Call(uint256.NewInt(0xffff), common.HexToAddress("0x1337"), big.NewInt(1), 1, 2, 3, 4).Hex() + want := "6004600360026001600161133761fffff1" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestMstore(t *testing.T) { + { + have := New().Mstore(common.FromHex("0xaabb"), 0).Hex() + want := "60aa60005360bb600153" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // store at offset + have := New().Mstore(common.FromHex("0xaabb"), 3).Hex() + want := "60aa60035360bb600453" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // 34 bytes + data := common.FromHex("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" + + "FFFF") + + have := New().Mstore(data, 0).Hex() + want := "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260ff60205360ff602153" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestMemToStorage(t *testing.T) { + have := New().MemToStorage(0, 33, 1).Hex() + want := "600051600155602051600255" + if have != want { + t.Errorf("have %v want %v", have, want) + } +} + +func TestSstore(t *testing.T) { + have := New().Sstore(0x1337, []byte("1234")).Hex() + want := "633132333461133755" + if have != want { + t.Errorf("have %v want %v", have, want) + } +} + +func TestReturnData(t *testing.T) { + { + have := New().ReturnData([]byte{0xFF}).Hex() + want := "60ff60005360016000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { + // 32 bytes + data := common.FromHex("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF") + have := New().ReturnData(data).Hex() + want := "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // ReturnViaCodeCopy + data := common.FromHex("0x6001") + have := New().Append([]byte{0x5b, 0x5b, 0x5b}).ReturnViaCodeCopy(data).Hex() + want := "5b5b5b600261001060003960026000f36001" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } + { // ReturnViaCodeCopy larger code + data := common.FromHex("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3") + have := New().Append([]byte{0x5b, 0x5b, 0x5b}).ReturnViaCodeCopy(data).Hex() + want := "5b5b5b602961001060003960296000f37fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff60005260206000f3" + if have != want { + t.Errorf("have %v want %v", have, want) + } + } +} + +func TestCreateAndCall(t *testing.T) { + // A constructor that stores a slot + ctor := New().Sstore(0, big.NewInt(5)) + + // A runtime bytecode which reads the slot and returns + deployed := New() + deployed.Push(0).Op(vm.SLOAD) // [value] in stack + deployed.Push(0) // [value, 0] + deployed.Op(vm.MSTORE) + deployed.Return(0, 32) + + // Pack them + ctor.ReturnData(deployed.Bytes()) + // Verify constructor + runtime code + { + want := "6005600055606060005360006001536054600253606060035360006004536052600553606060065360206007536060600853600060095360f3600a53600b6000f3" + if got := ctor.Hex(); got != want { + t.Fatalf("1: got %v expected %v", got, want) + } + } +} + +func TestCreate2Call(t *testing.T) { + // Some runtime code + runtime := New().Op(vm.ADDRESS, vm.SELFDESTRUCT).Bytes() + want := common.FromHex("0x30ff") + if !bytes.Equal(want, runtime) { + t.Fatalf("runtime code error\nwant: %x\nhave: %x\n", want, runtime) + } + // A constructor returning the runtime code + initcode := New().ReturnData(runtime).Bytes() + want = common.FromHex("603060005360ff60015360026000f3") + if !bytes.Equal(want, initcode) { + t.Fatalf("initcode error\nwant: %x\nhave: %x\n", want, initcode) + } + // A factory invoking the constructor + outer := New().Create2ThenCall(initcode, nil).Bytes() + want = common.FromHex("60606000536030600153606060025360006003536053600453606060055360ff6006536060600753600160085360536009536060600a536002600b536060600c536000600d5360f3600e536000600f60006000f560006000600060006000855af15050") + if !bytes.Equal(want, outer) { + t.Fatalf("factory error\nwant: %x\nhave: %x\n", want, outer) + } +} + +func TestGenerator(t *testing.T) { + for i, tc := range []struct { + want []byte + haveFn func() []byte + }{ + { // CREATE + want: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // length, offset, value + byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE), + byte(vm.POP), + }, + haveFn: func() []byte { + initcode := New().Return(0, 0).Bytes() + return New().MstoreSmall(initcode, 0). + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE). + Op(vm.POP).Bytes() + }, + }, + { // CREATE2 + want: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // salt, length, offset, value + byte(vm.PUSH1), 1, byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE2), + byte(vm.POP), + }, + haveFn: func() []byte { + initcode := New().Return(0, 0).Bytes() + return New().MstoreSmall(initcode, 0). + Push(1). // salt + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE2). + Op(vm.POP).Bytes() + }, + }, + { // CALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.DUP1), // value + byte(vm.PUSH1), 0xbb, //address + byte(vm.GAS), // gas + byte(vm.CALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().Call(nil, 0xbb, 0, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // CALLCODE + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0, // value + byte(vm.PUSH1), 0xcc, //address + byte(vm.GAS), // gas + byte(vm.CALLCODE), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().CallCode(nil, 0xcc, 0, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // STATICCALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xdd, //address + byte(vm.GAS), // gas + byte(vm.STATICCALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().StaticCall(nil, 0xdd, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + { // DELEGATECALL + want: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.DUP1), byte(vm.DUP1), byte(vm.DUP1), + byte(vm.PUSH1), 0xee, //address + byte(vm.GAS), // gas + byte(vm.DELEGATECALL), + byte(vm.POP), + }, + haveFn: func() []byte { + return New().DelegateCall(nil, 0xee, 0, 0, 0, 0).Op(vm.POP).Bytes() + }, + }, + } { + if have := tc.haveFn(); !bytes.Equal(have, tc.want) { + t.Fatalf("test %d error\nhave: %x\nwant: %x\n", i, have, tc.want) + } + } +} diff --git a/core/vm/program/readme.md b/core/vm/program/readme.md new file mode 100644 index 0000000000..0e4a54d8f1 --- /dev/null +++ b/core/vm/program/readme.md @@ -0,0 +1,30 @@ +### What is this + +In many cases, we have a need to create somewhat nontrivial bytecode, for testing various +quirks related to state transition or evm execution. + +For example, we want to have a `CREATE2`- op create a contract, which is then invoked, and when invoked does a selfdestruct-to-self. + +It is overkill to go full solidity, but it is also a bit tricky do assemble this by concatenating bytes. + +This utility takes an approach from [goevmlab](https://github.com/holiman/goevmlab/) where it has been used for several years, +a go-lang utility to assemble evm bytecode. + +Using this utility, the case above can be expressed as: +```golang + // Some runtime code + runtime := program.New().Ops(vm.ADDRESS, vm.SELFDESTRUCT).Bytecode() + // A constructor returning the runtime code + initcode := program.New().ReturnData(runtime).Bytecode() + // A factory invoking the constructor + outer := program.New().Create2AndCall(initcode, nil).Bytecode() +``` + +### Warning + +This package is a utility for testing, _not_ for production. As such: + +- There are not package guarantees. We might iterate heavily on this package, and do backwards-incompatible changes without warning +- There are no quality-guarantees. These utilities may produce evm-code that is non-functional. YMMV. +- There are no stability-guarantees. The utility will `panic` if the inputs do not align / make sense. + diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index 34335b8e9e..e54041f7e2 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -42,5 +42,7 @@ func NewEnv(cfg *Config) *vm.EVM { Random: cfg.Random, } - return vm.NewEVM(blockContext, txContext, cfg.State, cfg.ChainConfig, cfg.EVMConfig) + evm := vm.NewEVM(blockContext, cfg.State, cfg.ChainConfig, cfg.EVMConfig) + evm.SetTxContext(txContext) + return evm } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 97234368ee..7d1345a57b 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/vm/program" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/params" @@ -436,110 +437,46 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG // 55 ms func BenchmarkSimpleLoop(b *testing.B) { - staticCallIdentity := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas - byte(vm.STATICCALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl := program.New().Jumpdest() + // Call identity, and pop return value + staticCallIdentity := p. + StaticCall(nil, 0x4, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callIdentity := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callIdentity := p. + Call(nil, 0x4, 0, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callInexistant := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0xff, // address of existing contract - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callInexistant := p. + Call(nil, 0xff, 0, 0, 0, 0, 0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - callEOA := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.DUP1), // value - byte(vm.PUSH1), 0xE0, // address of EOA - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callEOA := p. + Call(nil, 0xE0, 0, 0, 0, 0, 0). // call addr of EOA + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label - loopingCode := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.DUP1), // out insize - byte(vm.DUP1), // in offset - byte(vm.PUSH1), 0x4, // address of identity - byte(vm.GAS), // gas + p, lbl = program.New().Jumpdest() + // Push as if we were making call, then pop it off again, and loop + loopingCode := p.Push(0). + Op(vm.DUP1, vm.DUP1, vm.DUP1). + Push(0x4). + Op(vm.GAS, vm.POP, vm.POP, vm.POP, vm.POP, vm.POP, vm.POP). + Jump(lbl).Bytes() - byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), byte(vm.POP), - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + loopingCode2 := p. + Push(0x01020304).Push(uint64(0x0102030405)). + Op(vm.POP, vm.POP). + Op(vm.PUSH6).Append(make([]byte, 6)).Op(vm.JUMP). // Jumpdest zero expressed in 6 bytes + Jump(lbl).Bytes() - loopingCode2 := []byte{ - byte(vm.JUMPDEST), // [ count ] - // push args for the call - byte(vm.PUSH4), 1, 2, 3, 4, - byte(vm.PUSH5), 1, 2, 3, 4, 5, - - byte(vm.POP), byte(vm.POP), - byte(vm.PUSH6), 0, 0, 0, 0, 0, 0, // jumpdestination - byte(vm.JUMP), - } - - callRevertingContractWithInput := []byte{ - byte(vm.JUMPDEST), // - // push args for the call - byte(vm.PUSH1), 0, // out size - byte(vm.DUP1), // out offset - byte(vm.PUSH1), 0x20, // in size - byte(vm.PUSH1), 0x00, // in offset - byte(vm.PUSH1), 0x00, // value - byte(vm.PUSH1), 0xEE, // address of reverting contract - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), // pop return value - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } + p, lbl = program.New().Jumpdest() + callRevertingContractWithInput := p. + Call(nil, 0xee, 0, 0, 0x20, 0x0, 0x0). + Op(vm.POP).Jump(lbl).Bytes() // pop return value and jump to label //tracer := logger.NewJSONLogger(nil, os.Stdout) //Execute(loopingCode, nil, &Config{ @@ -778,104 +715,49 @@ func TestRuntimeJSTracer(t *testing.T) { this.exits++; this.gasUsed = res.getGasUsed(); }}`} + initcode := program.New().Return(0, 0).Bytes() tests := []struct { code []byte // One result per tracer results []string }{ - { - // CREATE - code: []byte{ - // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) - byte(vm.PUSH5), - // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), - byte(vm.PUSH1), 0, - byte(vm.MSTORE), - // length, offset, value - byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, - byte(vm.CREATE), - byte(vm.POP), - }, + { // CREATE + code: program.New().MstoreSmall(initcode, 0). + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE). + Op(vm.POP).Bytes(), results: []string{`"1,1,952853,6,12"`, `"1,1,952853,6,0"`}, }, - { - // CREATE2 - code: []byte{ - // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) - byte(vm.PUSH5), - // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), - byte(vm.PUSH1), 0, - byte(vm.MSTORE), - // salt, length, offset, value - byte(vm.PUSH1), 1, byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, - byte(vm.CREATE2), - byte(vm.POP), - }, + { // CREATE2 + code: program.New().MstoreSmall(initcode, 0). + Push(1). // salt + Push(len(initcode)). // length + Push(32 - len(initcode)). // offset + Push(0). // value + Op(vm.CREATE2). + Op(vm.POP).Bytes(), results: []string{`"1,1,952844,6,13"`, `"1,1,952844,6,0"`}, }, - { - // CALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xbb, //address - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), - }, + { // CALL + code: program.New().Call(nil, 0xbb, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981796,6,13"`, `"1,1,981796,6,0"`}, }, - { - // CALLCODE - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xcc, //address - byte(vm.GAS), // gas - byte(vm.CALLCODE), - byte(vm.POP), - }, + { // CALLCODE + code: program.New().CallCode(nil, 0xcc, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981796,6,13"`, `"1,1,981796,6,0"`}, }, - { - // STATICCALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0xdd, //address - byte(vm.GAS), // gas - byte(vm.STATICCALL), - byte(vm.POP), - }, + { // STATICCALL + code: program.New().StaticCall(nil, 0xdd, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981799,6,12"`, `"1,1,981799,6,0"`}, }, - { - // DELEGATECALL - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0xee, //address - byte(vm.GAS), // gas - byte(vm.DELEGATECALL), - byte(vm.POP), - }, + { // DELEGATECALL + code: program.New().DelegateCall(nil, 0xee, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"1,1,981799,6,12"`, `"1,1,981799,6,0"`}, }, - { - // CALL self-destructing contract - code: []byte{ - // outsize, outoffset, insize, inoffset - byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, // value - byte(vm.PUSH1), 0xff, //address - byte(vm.GAS), // gas - byte(vm.CALL), - byte(vm.POP), - }, + { // CALL self-destructing contract + code: program.New().Call(nil, 0xff, 0, 0, 0, 0, 0).Op(vm.POP).Bytes(), results: []string{`"2,2,0,5003,12"`, `"2,2,0,5003,0"`}, }, } @@ -958,16 +840,8 @@ func TestJSTracerCreateTx(t *testing.T) { func BenchmarkTracerStepVsCallFrame(b *testing.B) { // Simply pushes and pops some values in a loop - code := []byte{ - byte(vm.JUMPDEST), - byte(vm.PUSH1), 0, - byte(vm.PUSH1), 0, - byte(vm.POP), - byte(vm.POP), - byte(vm.PUSH1), 0, // jumpdestination - byte(vm.JUMP), - } - + p, lbl := program.New().Jumpdest() + code := p.Push(0).Push(0).Op(vm.POP, vm.POP).Jump(lbl).Bytes() stepTracer := ` { step: function() {}, diff --git a/eth/api_backend.go b/eth/api_backend.go index 4e81d68e07..be2101c6ec 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -249,18 +249,17 @@ func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { +func (b *EthAPIBackend) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.eth.blockchain.GetVMConfig() } - txContext := core.NewEVMTxContext(msg) var context vm.BlockContext if blockCtx != nil { context = *blockCtx } else { context = core.NewEVMBlockContext(header, b.eth.BlockChain(), nil) } - return vm.NewEVM(context, txContext, state, b.ChainConfig(), *vmConfig) + return vm.NewEVM(context, state, b.ChainConfig(), *vmConfig) } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index db46afc30d..a24ff52101 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -21,7 +21,6 @@ import ( "crypto/sha256" "errors" "fmt" - "math/big" "sync" "time" @@ -34,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -287,12 +285,7 @@ func (c *SimulatedBeacon) Commit() common.Hash { // Rollback un-sends previously added transactions. func (c *SimulatedBeacon) Rollback() { - // Flush all transactions from the transaction pools - maxUint256 := new(big.Int).Sub(new(big.Int).Lsh(common.Big1, 256), common.Big1) - c.eth.TxPool().SetGasTip(maxUint256) - // Set the gas tip back to accept new transactions - // TODO (Marius van der Wijden): set gas tip to parameter passed by config - c.eth.TxPool().SetGasTip(big.NewInt(params.GWei)) + c.eth.TxPool().Clear() } // Fork sets the head to the provided hash. diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f1a815e6d3..8542bc97c4 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" ) @@ -160,7 +161,8 @@ type Config struct { // only exist on already merged networks. func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (consensus.Engine, error) { if config.TerminalTotalDifficulty == nil { - return nil, fmt.Errorf("only PoS networks are supported, please transition old ones with Geth v1.13.x") + log.Error("Geth only supports PoS networks. Please transition legacy networks using Geth v1.13.x.") + return nil, fmt.Errorf("'terminalTotalDifficulty' is not set in genesis block") } // Wrap previously supported consensus engines into their post-merge counterpart if config.Clique != nil { diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index d43057dda2..5729f84278 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -230,7 +230,8 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio if msgContext.BlobFeeCap != nil && msgContext.BlobFeeCap.BitLen() == 0 { evmContext.BlobBaseFee = new(big.Int) } - evm := vm.NewEVM(evmContext, msgContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) + evm := vm.NewEVM(evmContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) + evm.SetTxContext(msgContext) // Monitor the outer context and interrupt the EVM upon cancellation. To avoid // a dangling goroutine until the outer estimation finishes, create an internal // context for the lifetime of this method call. diff --git a/eth/state_accessor.go b/eth/state_accessor.go index cb5a233a83..43432cff31 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -235,16 +235,14 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, return nil, vm.BlockContext{}, nil, nil, err } // Insert parent beacon block root in the state as per EIP-4788. + context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) + evm := vm.NewEVM(context, statedb, eth.blockchain.Config(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, eth.blockchain.Config(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // If prague hardfork, insert parent block hash in the state as per EIP-2935. if eth.blockchain.Config().IsPrague(block.Number(), block.Time()) { - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, eth.blockchain.Config(), vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } if txIndex == 0 && len(block.Transactions()) == 0 { return nil, vm.BlockContext{}, statedb, release, nil @@ -252,22 +250,22 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // Recompute transactions up to the target index. signer := types.MakeSigner(eth.blockchain.Config(), block.Number(), block.Time()) for idx, tx := range block.Transactions() { - // Assemble the transaction call message and return if the requested offset - msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) if idx == txIndex { return tx, context, statedb, release, nil } + // Assemble the transaction call message and return if the requested offset + msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) + txContext := core.NewEVMTxContext(msg) + evm.SetTxContext(txContext) + // Not yet the searched for transaction, execute on top of the current state - vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) statedb.SetTxContext(tx.Hash(), idx) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 7d8191c25b..a2c11e0fe2 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -378,16 +378,14 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed } // Insert block's parent beacon block root in the state // as per EIP-4788. + context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(context, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := next.BeaconRoot(); beaconRoot != nil { - context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } // Insert parent hash in history contract. if api.backend.ChainConfig().IsPrague(next.Number(), next.Time()) { - context := core.NewEVMBlockContext(next.Header(), api.chainContext(ctx), nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessParentBlockHash(next.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(next.ParentHash(), evm) } // Clean out any pending release functions of trace state. Note this // step must be done after constructing tracing state, because the @@ -537,12 +535,12 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) deleteEmptyObjects = chainConfig.IsEIP158(block.Number()) ) + evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - core.ProcessParentBlockHash(block.ParentHash(), vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}), statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { if err := ctx.Err(); err != nil { @@ -551,10 +549,10 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config var ( msg, _ = core.TransactionToMessage(tx, signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) - vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) ) + evm.SetTxContext(txContext) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to @@ -605,13 +603,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac defer release() blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(blockCtx, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if api.backend.ChainConfig().IsPrague(block.Number(), block.Time()) { - vmenv := vm.NewEVM(blockCtx, vm.TxContext{}, statedb, api.backend.ChainConfig(), vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } // JS tracers have high overhead. In this case run a parallel @@ -695,6 +692,8 @@ func (api *API) traceBlockParallel(ctx context.Context, block *types.Block, stat // Feed the transactions into the tracers and return var failed error blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + evm := vm.NewEVM(blockCtx, statedb, api.backend.ChainConfig(), vm.Config{}) + txloop: for i, tx := range txs { // Send the trace task over for execution @@ -709,14 +708,14 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + evm.SetTxContext(core.NewEVMTxContext(msg)) + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { failed = err break txloop } // Finalize the state so any modifications are written to the trie // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } close(jobs) @@ -783,13 +782,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Note: This copies the config, to not screw up the main config chainConfig, canon = overrideConfig(chainConfig, config.Overrides) } + evm := vm.NewEVM(vmctx, statedb, chainConfig, vm.Config{}) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm) } if chainConfig.IsPrague(block.Number(), block.Time()) { - vmenv := vm.NewEVM(vmctx, vm.TxContext{}, statedb, chainConfig, vm.Config{}) - core.ProcessParentBlockHash(block.ParentHash(), vmenv, statedb) + core.ProcessParentBlockHash(block.ParentHash(), evm) } for i, tx := range block.Transactions() { // Prepare the transaction for un-traced execution @@ -822,12 +820,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } // Execute the transaction and flush any traces to disk - vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) + evm.SetTxContext(txContext) statedb.SetTxContext(tx.Hash(), i) if vmConf.Tracer.OnTxStart != nil { - vmConf.Tracer.OnTxStart(vmenv.GetVMContext(), tx, msg.From) + vmConf.Tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - vmRet, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) + vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) if vmConf.Tracer.OnTxEnd != nil { vmConf.Tracer.OnTxEnd(&types.Receipt{GasUsed: vmRet.UsedGas}, err) } @@ -843,7 +841,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } // Finalize the state so any modifications are written to the trie // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) // If we've traced the transaction we were looking for, abort if tx.Hash() == txHash { @@ -1017,7 +1015,8 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor } } // The actual TxContext will be created as part of ApplyTransactionWithEVM. - vmenv := vm.NewEVM(vmctx, vm.TxContext{GasPrice: message.GasPrice, BlobFeeCap: message.BlobGasFeeCap}, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + evm := vm.NewEVM(vmctx, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + evm.SetTxContext(vm.TxContext{GasPrice: message.GasPrice, BlobFeeCap: message.BlobGasFeeCap}) // Define a meaningful timeout of a single transaction trace if config.Timeout != nil { @@ -1031,14 +1030,14 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor if errors.Is(deadlineCtx.Err(), context.DeadlineExceeded) { tracer.Stop(errors.New("execution timeout")) // Stop evm execution. Note cancellation is not necessarily immediate. - vmenv.Cancel() + evm.Cancel() } }() defer cancel() // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, api.backend.ChainConfig(), new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, vmenv) + _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, tx, &usedGas, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 47e3693495..2b5cfa35c8 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -170,18 +170,19 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block } // Recompute transactions up to the target index. signer := types.MakeSigner(b.chainConfig, block.Number(), block.Time()) + context := core.NewEVMBlockContext(block.Header(), b.chain, nil) + evm := vm.NewEVM(context, statedb, b.chainConfig, vm.Config{}) for idx, tx := range block.Transactions() { - msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), b.chain, nil) if idx == txIndex { return tx, context, statedb, release, nil } - vmenv := vm.NewEVM(context, txContext, statedb, b.chainConfig, vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) + txContext := core.NewEVMTxContext(msg) + evm.SetTxContext(txContext) + if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } - statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) + statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) } return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cb635e7127..869558c324 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -132,7 +132,8 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { @@ -220,12 +221,15 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() + + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) + if err != nil { + b.Fatalf("failed to create call tracer: %v", err) + } + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(txContext) + for i := 0; i < b.N; i++ { - tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) - if err != nil { - b.Fatalf("failed to create call tracer: %v", err) - } - evm := vm.NewEVM(context, txContext, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) snap := state.StateDB.Snapshot() st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if _, err = st.TransitionDb(); err != nil { @@ -372,7 +376,8 @@ func TestInternals(t *testing.T) { Origin: origin, GasPrice: tx.GasPrice(), } - evm := vm.NewEVM(context, txContext, logState, config, vm.Config{Tracer: tc.tracer.Hooks}) + evm := vm.NewEVM(context, logState, config, vm.Config{Tracer: tc.tracer.Hooks}) + evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, big.NewInt(0)) if err != nil { t.Fatalf("test %v: failed to create message: %v", tc.name, err) diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 0ec3c367bc..1dbdc7caac 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -98,7 +98,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string if err != nil { return fmt.Errorf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index c6cf10a483..b8dec51db1 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -106,7 +106,8 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } - evm := vm.NewEVM(context, core.NewEVMTxContext(msg), state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index ed2789d70d..54f628b5f3 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -64,20 +64,21 @@ func testCtx() *vmContext { func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { var ( - env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer.Hooks}) + evm = vm.NewEVM(vmctx.blockCtx, &dummyStatedb{}, chaincfg, vm.Config{Tracer: tracer.Hooks}) gasLimit uint64 = 31000 startGas uint64 = 10000 value = uint256.NewInt(0) contract = vm.NewContract(account{}, account{}, value, startGas) ) + evm.SetTxContext(vmctx.txCtx) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} if contractCode != nil { contract.Code = contractCode } - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) - ret, err := env.Interpreter().Run(contract, []byte{}, false) + ret, err := evm.Interpreter().Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) // Rest gas assumes no refund tracer.OnTxEnd(&types.Receipt{GasUsed: gasLimit - contract.Gas}, nil) @@ -191,8 +192,9 @@ func TestHaltBetweenSteps(t *testing.T) { scope := &vm.ScopeContext{ Contract: vm.NewContract(&account{}, &account{}, uint256.NewInt(0), 0), } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(1)}) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 0, big.NewInt(0)) tracer.OnOpcode(0, 0, 0, 0, scope, nil, 0, nil) timeout := errors.New("stahp") @@ -214,8 +216,9 @@ func TestNoStepExec(t *testing.T) { if err != nil { t.Fatal(err) } - env := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, vm.TxContext{GasPrice: big.NewInt(100)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) - tracer.OnTxStart(env.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) + evm := vm.NewEVM(vm.BlockContext{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, chainConfig, vm.Config{Tracer: tracer.Hooks}) + evm.SetTxContext(vm.TxContext{GasPrice: big.NewInt(100)}) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{}), common.Address{}) tracer.OnEnter(0, byte(vm.CALL), common.Address{}, common.Address{}, []byte{}, 1000, big.NewInt(0)) tracer.OnExit(0, nil, 0, nil, false) ret, err := tracer.GetResult() diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index f918ce154b..bdd4e07055 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -363,26 +363,35 @@ func (t *mdLogger) OnEnter(depth int, typ byte, from common.Address, to common.A if depth != 0 { return } - create := vm.OpCode(typ) == vm.CREATE - if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + if create := vm.OpCode(typ) == vm.CREATE; !create { + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - to: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", - from.String(), to.String(), - input, gas, value) + fmt.Fprintf(t.out, "Pre-execution info:\n"+ + " - from: `%v`\n"+ + " - create: `%v`\n"+ + " - data: `%#x`\n"+ + " - gas: `%d`\n"+ + " - value: `%v` wei\n", + from.String(), to.String(), input, gas, value) } - fmt.Fprintf(t.out, ` -| Pc | Op | Cost | Stack | RStack | Refund | -|-------|-------------|------|-----------|-----------|---------| +| Pc | Op | Cost | Refund | Stack | +|-------|-------------|------|-----------|-----------| `) } func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { if depth == 0 { - fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nPost-execution info:\n"+ + " - output: `%#x`\n"+ + " - consumed gas: `%d`\n"+ + " - error: `%v`\n", output, gasUsed, err) } } @@ -390,7 +399,8 @@ func (t *mdLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, r // OnOpcode also tracks SLOAD/SSTORE ops to track storage change. func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { stack := scope.StackData() - fmt.Fprintf(t.out, "| %4d | %10v | %3d |", pc, vm.OpCode(op).String(), cost) + fmt.Fprintf(t.out, "| %4d | %10v | %3d |%10v |", pc, vm.OpCode(op).String(), + cost, t.env.StateDB.GetRefund()) if !t.cfg.DisableStack { // format stack @@ -401,7 +411,6 @@ func (t *mdLogger) OnOpcode(pc uint64, op byte, gas, cost uint64, scope tracing. b := fmt.Sprintf("[%v]", strings.Join(a, ",")) fmt.Fprintf(t.out, "%10v |", b) } - fmt.Fprintf(t.out, "%10v |", t.env.StateDB.GetRefund()) fmt.Fprintln(t.out, "") if err != nil { fmt.Fprintf(t.out, "Error: %v\n", err) diff --git a/eth/tracers/logger/logger_json.go b/eth/tracers/logger/logger_json.go index 797f7ac658..52ac3945d4 100644 --- a/eth/tracers/logger/logger_json.go +++ b/eth/tracers/logger/logger_json.go @@ -71,7 +71,7 @@ func NewJSONLogger(cfg *Config, writer io.Writer) *tracing.Hooks { l.hooks = &tracing.Hooks{ OnTxStart: l.OnTxStart, OnSystemCallStart: l.onSystemCallStart, - OnExit: l.OnEnd, + OnExit: l.OnExit, OnOpcode: l.OnOpcode, OnFault: l.OnFault, } @@ -152,13 +152,6 @@ func (l *jsonLogger) OnEnter(depth int, typ byte, from common.Address, to common l.encoder.Encode(frame) } -func (l *jsonLogger) OnEnd(depth int, output []byte, gasUsed uint64, err error, reverted bool) { - if depth > 0 { - return - } - l.OnExit(depth, output, gasUsed, err, false) -} - func (l *jsonLogger) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { type endLog struct { Output string `json:"output"` diff --git a/eth/tracers/logger/logger_test.go b/eth/tracers/logger/logger_test.go index fb1a0154e3..e7799dde35 100644 --- a/eth/tracers/logger/logger_test.go +++ b/eth/tracers/logger/logger_test.go @@ -58,13 +58,13 @@ func (*dummyStatedb) SetState(_ common.Address, _ common.Hash, _ common.Hash) co func TestStoreCapture(t *testing.T) { var ( logger = NewStructLogger(nil) - env = vm.NewEVM(vm.BlockContext{}, vm.TxContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger.Hooks()}) + evm = vm.NewEVM(vm.BlockContext{}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Tracer: logger.Hooks()}) contract = vm.NewContract(&dummyContractRef{}, &dummyContractRef{}, new(uint256.Int), 100000) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, byte(vm.SSTORE)} var index common.Hash - logger.OnTxStart(env.GetVMContext(), nil, common.Address{}) - _, err := env.Interpreter().Run(contract, []byte{}, false) + logger.OnTxStart(evm.GetVMContext(), nil, common.Address{}) + _, err := evm.Interpreter().Run(contract, []byte{}, false) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 31e14b9112..72cc1bde25 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -89,7 +89,8 @@ func BenchmarkTransactionTrace(b *testing.B) { //EnableMemory: false, //EnableReturnData: false, }) - evm := vm.NewEVM(context, txContext, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) + evm := vm.NewEVM(context, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) + evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) diff --git a/go.mod b/go.mod index 253c55c7f9..1928f7b5c5 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 github.com/karalabe/hid v1.0.1-0.20240306101548-573246063e52 - github.com/kilic/bls12-381 v0.1.0 github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 @@ -123,6 +122,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a508b0ca5b..ded4db50aa 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "errors" "fmt" - "maps" gomath "math" "math/big" "strings" @@ -254,7 +253,7 @@ func (api *TxPoolAPI) Inspect() map[string]map[string]map[string]string { pending, queue := api.b.TxPoolContent() // Define a formatter to flatten a transaction into a string - var format = func(tx *types.Transaction) string { + format := func(tx *types.Transaction) string { if to := tx.To(); to != nil { return fmt.Sprintf("%s: %v wei + %v gas × %v wei", tx.To().Hex(), tx.Value(), tx.Gas(), tx.GasPrice()) } @@ -825,7 +824,7 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S blockOverrides.Apply(&blockCtx) } rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time) - precompiles := maps.Clone(vm.ActivePrecompiledContracts(rules)) + precompiles := vm.ActivePrecompiledContracts(rules) if err := overrides.Apply(state, precompiles); err != nil { return nil, err } @@ -864,10 +863,11 @@ func applyMessage(ctx context.Context, b Backend, args TransactionArgs, state *s if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { blockContext.BlobBaseFee = new(big.Int) } - evm := b.GetEVM(ctx, msg, state, header, vmConfig, blockContext) + evm := b.GetEVM(ctx, state, header, vmConfig, blockContext) if precompiles != nil { evm.SetPrecompiles(precompiles) } + evm.SetTxContext(core.NewEVMTxContext(msg)) res, err := applyMessageWithEVM(ctx, evm, msg, timeout, gp) // If an internal state error occurred, let that have precedence. Otherwise, // a "trie root missing" type of error will masquerade as e.g. "insufficient gas" @@ -1331,7 +1331,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Apply the transaction with the access list tracer tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} - vmenv := b.GetEVM(ctx, msg, statedb, header, &config, nil) + vmenv := b.GetEVM(ctx, statedb, header, &config, nil) // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). if msg.GasPrice.Sign() == 0 { @@ -1340,6 +1340,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { vmenv.Context.BlobBaseFee = new(big.Int) } + vmenv.SetTxContext(core.NewEVMTxContext(msg)) res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) @@ -1760,11 +1761,11 @@ func (api *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, matchTx := sendArgs.ToTransaction(types.LegacyTxType) // Before replacing the old transaction, ensure the _new_ transaction fee is reasonable. - var price = matchTx.GasPrice() + price := matchTx.GasPrice() if gasPrice != nil { price = gasPrice.ToInt() } - var gas = matchTx.Gas() + gas := matchTx.Gas() if gasLimit != nil { gas = uint64(*gasLimit) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 4b24cf958a..e0d66c1f2f 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -570,16 +570,15 @@ func (b testBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { } return big.NewInt(1) } -func (b testBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) *vm.EVM { +func (b testBackend) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.chain.GetVMConfig() } - txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, b.chain, nil) if blockContext != nil { context = *blockContext } - return vm.NewEVM(context, txContext, state, b.chain.Config(), *vmConfig) + return vm.NewEVM(context, state, b.chain.Config(), *vmConfig) } func (b testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { panic("implement me") diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 82465ca7d7..3d8f7aa700 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -68,7 +68,7 @@ type Backend interface { StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) Pending() (*types.Block, types.Receipts, *state.StateDB) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) - GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM + GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 81b4633d42..2161e1d5f4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "math/big" "time" @@ -185,12 +184,12 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, NoBaseFee: !sim.validate, Tracer: tracer.Hooks(), } - evm = vm.NewEVM(blockContext, vm.TxContext{GasPrice: new(big.Int)}, sim.state, sim.chainConfig, *vmConfig) ) - var tracingStateDB = vm.StateDB(sim.state) + tracingStateDB := vm.StateDB(sim.state) if hooks := tracer.Hooks(); hooks != nil { tracingStateDB = state.NewHookedState(sim.state, hooks) } + evm := vm.NewEVM(blockContext, tracingStateDB, sim.chainConfig, *vmConfig) // It is possible to override precompiles with EVM bytecode, or // move them to another address. if precompiles != nil { @@ -208,7 +207,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, tracer.reset(tx.Hash(), uint(i)) // EoA check is always skipped, even in validation mode. msg := call.ToMessage(header.BaseFee, !sim.validate, true) - evm.Reset(core.NewEVMTxContext(msg), tracingStateDB) + evm.SetTxContext(core.NewEVMTxContext(msg)) result, err := applyMessageWithEVM(ctx, evm, msg, timeout, sim.gp) if err != nil { txErr := txValidationError(err) @@ -265,7 +264,7 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state *state.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) @@ -289,7 +288,7 @@ func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContra isMerge = (base.Difficulty.Sign() == 0) rules = sim.chainConfig.Rules(base.Number, isMerge, base.Time) ) - return maps.Clone(vm.ActivePrecompiledContracts(rules)) + return vm.ActivePrecompiledContracts(rules) } // sanitizeChain checks the chain integrity. Specifically it checks that diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 5f59b491e1..7172fc883f 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -370,7 +370,7 @@ func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number return nil, nil } func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *backendMock) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { +func (b *backendMock) GetEVM(ctx context.Context, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { return nil } func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 1f6be3d3d3..b858e73d04 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -90,7 +90,7 @@ func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { } } eachName(f, func(name string) { - set.Var(&f.Value, f.Name, f.Usage) + set.Var(&f.Value, name, f.Usage) }) return nil } @@ -172,7 +172,7 @@ func (f *BigFlag) Apply(set *flag.FlagSet) error { } eachName(f, func(name string) { f.Value = new(big.Int) - set.Var((*bigValue)(f.Value), f.Name, f.Usage) + set.Var((*bigValue)(f.Value), name, f.Usage) }) return nil } diff --git a/miner/worker.go b/miner/worker.go index db2fac3871..aeb6cfcdc2 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -51,6 +51,7 @@ type environment struct { tcount int // tx count in cycle gasPool *core.GasPool // available gas used to pack transactions coinbase common.Address + evm *vm.EVM header *types.Header txs []*types.Transaction @@ -126,14 +127,11 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo return &newPayloadResult{err: err} } requests = append(requests, depositRequests) - // create EVM for system calls - blockContext := core.NewEVMBlockContext(work.header, miner.chain, &work.header.Coinbase) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, work.state, miner.chainConfig, vm.Config{}) // EIP-7002 withdrawals - withdrawalRequests := core.ProcessWithdrawalQueue(vmenv, work.state) + withdrawalRequests := core.ProcessWithdrawalQueue(work.evm) requests = append(requests, withdrawalRequests) // EIP-7251 consolidations - consolidationRequests := core.ProcessConsolidationQueue(vmenv, work.state) + consolidationRequests := core.ProcessConsolidationQueue(work.evm) requests = append(requests, consolidationRequests) } if requests != nil { @@ -233,14 +231,10 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir return nil, err } if header.ParentBeaconRoot != nil { - context := core.NewEVMBlockContext(header, miner.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) - core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, vmenv, env.state) + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, env.evm) } if miner.chainConfig.IsPrague(header.Number, header.Time) { - context := core.NewEVMBlockContext(header, miner.chain, nil) - vmenv := vm.NewEVM(context, vm.TxContext{}, env.state, miner.chainConfig, vm.Config{}) - core.ProcessParentBlockHash(header.ParentHash, vmenv, env.state) + core.ProcessParentBlockHash(header.ParentHash, env.evm) } return env, nil } @@ -266,6 +260,7 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase coinbase: coinbase, header: header, witness: state.Witness(), + evm: vm.NewEVM(core.NewEVMBlockContext(header, miner.chain, &coinbase), state, miner.chainConfig, vm.Config{}), }, nil } @@ -314,7 +309,7 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (* snap = env.state.Snapshot() gp = env.gasPool.Gas() ) - receipt, err := core.ApplyTransaction(miner.chainConfig, miner.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) if err != nil { env.state.RevertToSnapshot(snap) env.gasPool.SetGas(gp) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 50491b9155..5e4aa1c253 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -160,6 +160,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG1Add fuzz_g1_add\ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG1Mul fuzz_cross_g1_mul\ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG1Mul fuzz_g1_mul\ $repo/tests/fuzzers/bls12381/bls12381_test.go @@ -172,6 +176,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG2Add fuzz_g2_add \ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG2Mul fuzz_cross_g2_mul\ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzG2Mul fuzz_g2_mul\ $repo/tests/fuzzers/bls12381/bls12381_test.go @@ -204,6 +212,10 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzCrossG2Add fuzz_cross_g2_add \ $repo/tests/fuzzers/bls12381/bls12381_test.go +compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ + FuzzCrossG2MultiExp fuzz_cross_g2_multiexp \ + $repo/tests/fuzzers/bls12381/bls12381_test.go + compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/bls12381 \ FuzzCrossPairing fuzz_cross_pairing\ $repo/tests/fuzzers/bls12381/bls12381_test.go diff --git a/rpc/client_test.go b/rpc/client_test.go index 49f2350b40..6c1a4f8f6c 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -38,6 +38,8 @@ import ( ) func TestClientRequest(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -53,6 +55,8 @@ func TestClientRequest(t *testing.T) { } func TestClientResponseType(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -71,6 +75,8 @@ func TestClientResponseType(t *testing.T) { // This test checks calling a method that returns 'null'. func TestClientNullResponse(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -91,6 +97,8 @@ func TestClientNullResponse(t *testing.T) { // This test checks that server-returned errors with code and data come out of Client.Call. func TestClientErrorData(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -121,6 +129,8 @@ func TestClientErrorData(t *testing.T) { } func TestClientBatchRequest(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -172,6 +182,8 @@ func TestClientBatchRequest(t *testing.T) { // This checks that, for HTTP connections, the length of batch responses is validated to // match the request exactly. func TestClientBatchRequest_len(t *testing.T) { + t.Parallel() + b, err := json.Marshal([]jsonrpcMessage{ {Version: "2.0", ID: json.RawMessage("1"), Result: json.RawMessage(`"0x1"`)}, {Version: "2.0", ID: json.RawMessage("2"), Result: json.RawMessage(`"0x2"`)}, @@ -188,6 +200,8 @@ func TestClientBatchRequest_len(t *testing.T) { t.Cleanup(s.Close) t.Run("too-few", func(t *testing.T) { + t.Parallel() + client, err := Dial(s.URL) if err != nil { t.Fatal("failed to dial test server:", err) @@ -218,6 +232,8 @@ func TestClientBatchRequest_len(t *testing.T) { }) t.Run("too-many", func(t *testing.T) { + t.Parallel() + client, err := Dial(s.URL) if err != nil { t.Fatal("failed to dial test server:", err) @@ -249,6 +265,8 @@ func TestClientBatchRequest_len(t *testing.T) { // This checks that the client can handle the case where the server doesn't // respond to all requests in a batch. func TestClientBatchRequestLimit(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() server.SetBatchLimits(2, 100000) @@ -285,6 +303,8 @@ func TestClientBatchRequestLimit(t *testing.T) { } func TestClientNotify(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -392,6 +412,8 @@ func testClientCancel(transport string, t *testing.T) { } func TestClientSubscribeInvalidArg(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -422,6 +444,8 @@ func TestClientSubscribeInvalidArg(t *testing.T) { } func TestClientSubscribe(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() client := DialInProc(server) @@ -454,6 +478,8 @@ func TestClientSubscribe(t *testing.T) { // In this test, the connection drops while Subscribe is waiting for a response. func TestClientSubscribeClose(t *testing.T) { + t.Parallel() + server := newTestServer() service := ¬ificationTestService{ gotHangSubscriptionReq: make(chan struct{}), @@ -498,6 +524,8 @@ func TestClientSubscribeClose(t *testing.T) { // This test reproduces https://github.com/ethereum/go-ethereum/issues/17837 where the // client hangs during shutdown when Unsubscribe races with Client.Close. func TestClientCloseUnsubscribeRace(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -540,6 +568,8 @@ func (b *unsubscribeBlocker) readBatch() ([]*jsonrpcMessage, bool, error) { // not respond. // It reproducers the issue https://github.com/ethereum/go-ethereum/issues/30156 func TestUnsubscribeTimeout(t *testing.T) { + t.Parallel() + srv := NewServer() srv.RegisterName("nftest", new(notificationTestService)) @@ -674,6 +704,8 @@ func TestClientSubscriptionChannelClose(t *testing.T) { // This test checks that Client doesn't lock up when a single subscriber // doesn't read subscription events. func TestClientNotificationStorm(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -726,6 +758,8 @@ func TestClientNotificationStorm(t *testing.T) { } func TestClientSetHeader(t *testing.T) { + t.Parallel() + var gotHeader bool srv := newTestServer() httpsrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -762,6 +796,8 @@ func TestClientSetHeader(t *testing.T) { } func TestClientHTTP(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -804,6 +840,8 @@ func TestClientHTTP(t *testing.T) { } func TestClientReconnect(t *testing.T) { + t.Parallel() + startServer := func(addr string) (*Server, net.Listener) { srv := newTestServer() l, err := net.Listen("tcp", addr) diff --git a/rpc/http_test.go b/rpc/http_test.go index ad86ca15ae..6c268b6292 100644 --- a/rpc/http_test.go +++ b/rpc/http_test.go @@ -58,24 +58,34 @@ func confirmRequestValidationCode(t *testing.T, method, contentType, body string } func TestHTTPErrorResponseWithDelete(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodDelete, contentType, "", http.StatusMethodNotAllowed) } func TestHTTPErrorResponseWithPut(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPut, contentType, "", http.StatusMethodNotAllowed) } func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) { + t.Parallel() + body := make([]rune, defaultBodyLimit+1) confirmRequestValidationCode(t, http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge) } func TestHTTPErrorResponseWithEmptyContentType(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPost, "", "", http.StatusUnsupportedMediaType) } func TestHTTPErrorResponseWithValidRequest(t *testing.T) { + t.Parallel() + confirmRequestValidationCode(t, http.MethodPost, contentType, "", 0) } @@ -101,11 +111,15 @@ func confirmHTTPRequestYieldsStatusCode(t *testing.T, method, contentType, body } func TestHTTPResponseWithEmptyGet(t *testing.T) { + t.Parallel() + confirmHTTPRequestYieldsStatusCode(t, http.MethodGet, "", "", http.StatusOK) } // This checks that maxRequestContentLength is not applied to the response of a request. func TestHTTPRespBodyUnlimited(t *testing.T) { + t.Parallel() + const respLength = defaultBodyLimit * 3 s := NewServer() @@ -132,6 +146,8 @@ func TestHTTPRespBodyUnlimited(t *testing.T) { // Tests that an HTTP error results in an HTTPError instance // being returned with the expected attributes. func TestHTTPErrorResponse(t *testing.T) { + t.Parallel() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "error has occurred!", http.StatusTeapot) })) @@ -169,6 +185,8 @@ func TestHTTPErrorResponse(t *testing.T) { } func TestHTTPPeerInfo(t *testing.T) { + t.Parallel() + s := newTestServer() defer s.Stop() ts := httptest.NewServer(s) @@ -205,6 +223,8 @@ func TestHTTPPeerInfo(t *testing.T) { } func TestNewContextWithHeaders(t *testing.T) { + t.Parallel() + expectedHeaders := 0 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { for i := 0; i < expectedHeaders; i++ { diff --git a/rpc/server_test.go b/rpc/server_test.go index 9d1c7fb5f0..9ee545d81a 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -29,6 +29,8 @@ import ( ) func TestServerRegisterName(t *testing.T) { + t.Parallel() + server := NewServer() service := new(testService) @@ -53,6 +55,8 @@ func TestServerRegisterName(t *testing.T) { } func TestServer(t *testing.T) { + t.Parallel() + files, err := os.ReadDir("testdata") if err != nil { t.Fatal("where'd my testdata go?") @@ -64,6 +68,8 @@ func TestServer(t *testing.T) { path := filepath.Join("testdata", f.Name()) name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name())) t.Run(name, func(t *testing.T) { + t.Parallel() + runTestScript(t, path) }) } @@ -116,6 +122,8 @@ func runTestScript(t *testing.T, file string) { // This test checks that responses are delivered for very short-lived connections that // only carry a single request. func TestServerShortLivedConn(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() @@ -156,6 +164,8 @@ func TestServerShortLivedConn(t *testing.T) { } func TestServerBatchResponseSizeLimit(t *testing.T) { + t.Parallel() + server := newTestServer() defer server.Stop() server.SetBatchLimits(100, 60) diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index ab40ab169f..e52f390adb 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -33,6 +33,8 @@ import ( ) func TestNewID(t *testing.T) { + t.Parallel() + hexchars := "0123456789ABCDEFabcdef" for i := 0; i < 100; i++ { id := string(NewID()) @@ -54,6 +56,8 @@ func TestNewID(t *testing.T) { } func TestSubscriptions(t *testing.T) { + t.Parallel() + var ( namespaces = []string{"eth", "bzz"} service = ¬ificationTestService{} @@ -132,6 +136,8 @@ func TestSubscriptions(t *testing.T) { // This test checks that unsubscribing works. func TestServerUnsubscribe(t *testing.T) { + t.Parallel() + p1, p2 := net.Pipe() defer p2.Close() @@ -260,6 +266,8 @@ func BenchmarkNotify(b *testing.B) { } func TestNotify(t *testing.T) { + t.Parallel() + out := new(bytes.Buffer) id := ID("test") notifier := &Notifier{ diff --git a/rpc/types_test.go b/rpc/types_test.go index aba40b5863..9dd6fa6508 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -26,6 +26,8 @@ import ( ) func TestBlockNumberJSONUnmarshal(t *testing.T) { + t.Parallel() + tests := []struct { input string mustFail bool @@ -70,6 +72,8 @@ func TestBlockNumberJSONUnmarshal(t *testing.T) { } func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { + t.Parallel() + tests := []struct { input string mustFail bool @@ -131,6 +135,8 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { } func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { + t.Parallel() + tests := []struct { name string number int64 @@ -144,6 +150,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + t.Parallel() + bnh := BlockNumberOrHashWithNumber(BlockNumber(test.number)) marshalled, err := json.Marshal(bnh) if err != nil { @@ -162,6 +170,8 @@ func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { } func TestBlockNumberOrHash_StringAndUnmarshal(t *testing.T) { + t.Parallel() + tests := []BlockNumberOrHash{ BlockNumberOrHashWithNumber(math.MaxInt64), BlockNumberOrHashWithNumber(PendingBlockNumber), diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index c6ea325d29..10a998b351 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -174,6 +174,8 @@ func TestWebsocketLargeRead(t *testing.T) { } func TestWebsocketPeerInfo(t *testing.T) { + t.Parallel() + var ( s = newTestServer() ts = httptest.NewServer(s.WebsocketHandler([]string{"origin.example.com"})) @@ -259,6 +261,8 @@ func TestClientWebsocketPing(t *testing.T) { // This checks that the websocket transport can deal with large messages. func TestClientWebsocketLargeMessage(t *testing.T) { + t.Parallel() + var ( srv = NewServer() httpsrv = httptest.NewServer(srv.WebsocketHandler(nil)) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 2ae182279a..b56931c1d1 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -676,7 +676,7 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf if err != nil { return nil, err } - return math.U256Bytes(b), nil + return math.U256Bytes(new(big.Int).Set(b)), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) } diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index 74ea6f52a7..a3e0e9f72b 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -31,42 +31,33 @@ import ( "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" "github.com/ethereum/go-ethereum/common" - bls12381 "github.com/kilic/bls12-381" blst "github.com/supranational/blst/bindings/go" ) func fuzzG1SubgroupChecks(data []byte) int { input := bytes.NewReader(data) - kpG1, cpG1, blG1, err := getG1Points(input) + cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } - inSubGroupKilic := bls12381.NewG1().InCorrectSubgroup(kpG1) inSubGroupGnark := cpG1.IsInSubGroup() inSubGroupBLST := blG1.InG1() - if inSubGroupKilic != inSubGroupGnark { - panic(fmt.Sprintf("differing subgroup check, kilic %v, gnark %v", inSubGroupKilic, inSubGroupGnark)) - } - if inSubGroupKilic != inSubGroupBLST { - panic(fmt.Sprintf("differing subgroup check, kilic %v, blst %v", inSubGroupKilic, inSubGroupBLST)) + if inSubGroupGnark != inSubGroupBLST { + panic(fmt.Sprintf("differing subgroup check, gnark %v, blst %v", inSubGroupGnark, inSubGroupBLST)) } return 1 } func fuzzG2SubgroupChecks(data []byte) int { input := bytes.NewReader(data) - kpG2, cpG2, blG2, err := getG2Points(input) + gpG2, blG2, err := getG2Points(input) if err != nil { return 0 } - inSubGroupKilic := bls12381.NewG2().InCorrectSubgroup(kpG2) - inSubGroupGnark := cpG2.IsInSubGroup() + inSubGroupGnark := gpG2.IsInSubGroup() inSubGroupBLST := blG2.InG2() - if inSubGroupKilic != inSubGroupGnark { - panic(fmt.Sprintf("differing subgroup check, kilic %v, gnark %v", inSubGroupKilic, inSubGroupGnark)) - } - if inSubGroupKilic != inSubGroupBLST { - panic(fmt.Sprintf("differing subgroup check, kilic %v, blst %v", inSubGroupKilic, inSubGroupBLST)) + if inSubGroupGnark != inSubGroupBLST { + panic(fmt.Sprintf("differing subgroup check, gnark %v, blst %v", inSubGroupGnark, inSubGroupBLST)) } return 1 } @@ -75,38 +66,28 @@ func fuzzCrossPairing(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kpG1, cpG1, blG1, err := getG1Points(input) + cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } // get random G2 points - kpG2, cpG2, blG2, err := getG2Points(input) + cpG2, blG2, err := getG2Points(input) if err != nil { return 0 } - // compute pairing using geth - engine := bls12381.NewEngine() - engine.AddPair(kpG1, kpG2) - kResult := engine.Result() - // compute pairing using gnark cResult, err := gnark.Pair([]gnark.G1Affine{*cpG1}, []gnark.G2Affine{*cpG2}) if err != nil { panic(fmt.Sprintf("gnark/bls12381 encountered error: %v", err)) } - // compare result - if !(bytes.Equal(cResult.Marshal(), bls12381.NewGT().ToBytes(kResult))) { - panic("pairing mismatch gnark / geth ") - } - // compute pairing using blst blstResult := blst.Fp12MillerLoop(blG2, blG1) blstResult.FinalExp() res := massageBLST(blstResult.ToBendian()) - if !(bytes.Equal(res, bls12381.NewGT().ToBytes(kResult))) { + if !(bytes.Equal(res, cResult.Marshal())) { panic("pairing mismatch blst / geth") } @@ -141,32 +122,22 @@ func fuzzCrossG1Add(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kp1, cp1, bl1, err := getG1Points(input) + cp1, bl1, err := getG1Points(input) if err != nil { return 0 } // get random G1 points - kp2, cp2, bl2, err := getG1Points(input) + cp2, bl2, err := getG1Points(input) if err != nil { return 0 } - // compute kp = kp1 + kp2 - g1 := bls12381.NewG1() - kp := bls12381.PointG1{} - g1.Add(&kp, kp1, kp2) - // compute cp = cp1 + cp2 _cp1 := new(gnark.G1Jac).FromAffine(cp1) _cp2 := new(gnark.G1Jac).FromAffine(cp2) cp := new(gnark.G1Affine).FromJacobian(_cp1.AddAssign(_cp2)) - // compare result - if !(bytes.Equal(cp.Marshal(), g1.ToBytes(&kp))) { - panic("G1 point addition mismatch gnark / geth ") - } - bl3 := blst.P1AffinesAdd([]*blst.P1Affine{bl1, bl2}) if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { panic("G1 point addition mismatch blst / geth ") @@ -179,34 +150,24 @@ func fuzzCrossG2Add(data []byte) int { input := bytes.NewReader(data) // get random G2 points - kp1, cp1, bl1, err := getG2Points(input) + gp1, bl1, err := getG2Points(input) if err != nil { return 0 } // get random G2 points - kp2, cp2, bl2, err := getG2Points(input) + gp2, bl2, err := getG2Points(input) if err != nil { return 0 } - // compute kp = kp1 + kp2 - g2 := bls12381.NewG2() - kp := bls12381.PointG2{} - g2.Add(&kp, kp1, kp2) - // compute cp = cp1 + cp2 - _cp1 := new(gnark.G2Jac).FromAffine(cp1) - _cp2 := new(gnark.G2Jac).FromAffine(cp2) - cp := new(gnark.G2Affine).FromJacobian(_cp1.AddAssign(_cp2)) - - // compare result - if !(bytes.Equal(cp.Marshal(), g2.ToBytes(&kp))) { - panic("G2 point addition mismatch gnark / geth ") - } + _gp1 := new(gnark.G2Jac).FromAffine(gp1) + _gp2 := new(gnark.G2Jac).FromAffine(gp2) + gp := new(gnark.G2Affine).FromJacobian(_gp1.AddAssign(_gp2)) bl3 := blst.P2AffinesAdd([]*blst.P2Affine{bl1, bl2}) - if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + if !(bytes.Equal(gp.Marshal(), bl3.Serialize())) { panic("G1 point addition mismatch blst / geth ") } @@ -216,10 +177,10 @@ func fuzzCrossG2Add(data []byte) int { func fuzzCrossG1MultiExp(data []byte) int { var ( input = bytes.NewReader(data) - gethScalars []*bls12381.Fr gnarkScalars []fr.Element - gethPoints []*bls12381.PointG1 gnarkPoints []gnark.G1Affine + blstScalars []*blst.Scalar + blstPoints []*blst.P1Affine ) // n random scalars (max 17) for i := 0; i < 17; i++ { @@ -229,50 +190,147 @@ func fuzzCrossG1MultiExp(data []byte) int { break } // get a random G1 point as basis - kp1, cp1, _, err := getG1Points(input) + cp1, bl1, err := getG1Points(input) if err != nil { break } - gethScalars = append(gethScalars, bls12381.NewFr().FromBytes(s.Bytes())) - var gnarkScalar = &fr.Element{} - gnarkScalar = gnarkScalar.SetBigInt(s) - gnarkScalars = append(gnarkScalars, *gnarkScalar) - gethPoints = append(gethPoints, new(bls12381.PointG1).Set(kp1)) + gnarkScalar := new(fr.Element).SetBigInt(s) + gnarkScalars = append(gnarkScalars, *gnarkScalar) gnarkPoints = append(gnarkPoints, *cp1) + + blstScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + blstScalars = append(blstScalars, blstScalar) + blstPoints = append(blstPoints, bl1) } - if len(gethScalars) == 0 { + + if len(gnarkScalars) == 0 || len(gnarkScalars) != len(gnarkPoints) { return 0 } - // compute multi exponentiation - g1 := bls12381.NewG1() - kp := bls12381.PointG1{} - if _, err := g1.MultiExp(&kp, gethPoints, gethScalars); err != nil { - panic(fmt.Sprintf("G1 multi exponentiation errored (geth): %v", err)) - } - // note that geth/crypto/bls12381.MultiExp mutates the scalars slice (and sets all the scalars to zero) // gnark multi exp cp := new(gnark.G1Affine) cp.MultiExp(gnarkPoints, gnarkScalars, ecc.MultiExpConfig{}) - // compare result - gnarkRes := cp.Marshal() - gethRes := g1.ToBytes(&kp) - if !bytes.Equal(gnarkRes, gethRes) { - msg := fmt.Sprintf("G1 multi exponentiation mismatch gnark/geth.\ngnark: %x\ngeth: %x\ninput: %x\n ", - gnarkRes, gethRes, data) - panic(msg) + expectedGnark := multiExpG1Gnark(gnarkPoints, gnarkScalars) + if !bytes.Equal(cp.Marshal(), expectedGnark.Marshal()) { + panic("g1 multi exponentiation mismatch") } + // blst multi exp + expectedBlst := blst.P1AffinesMult(blstPoints, blstScalars, 256).ToAffine() + if !bytes.Equal(cp.Marshal(), expectedBlst.Serialize()) { + panic("g1 multi exponentiation mismatch, gnark/blst") + } return 1 } -func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1Affine, error) { +func fuzzCrossG1Mul(data []byte) int { + input := bytes.NewReader(data) + gp, blpAffine, err := getG1Points(input) + if err != nil { + return 0 + } + scalar, err := randomScalar(input, fp.Modulus()) + if err != nil { + return 0 + } + + blScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(scalar.Bytes(), 32)) + + blp := new(blst.P1) + blp.FromAffine(blpAffine) + + resBl := blp.Mult(blScalar) + resGeth := (new(gnark.G1Affine)).ScalarMultiplication(gp, scalar) + + if !bytes.Equal(resGeth.Marshal(), resBl.Serialize()) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + return 1 +} + +func fuzzCrossG2Mul(data []byte) int { + input := bytes.NewReader(data) + gp, blpAffine, err := getG2Points(input) + if err != nil { + return 0 + } + scalar, err := randomScalar(input, fp.Modulus()) + if err != nil { + return 0 + } + + blScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(scalar.Bytes(), 32)) + + blp := new(blst.P2) + blp.FromAffine(blpAffine) + + resBl := blp.Mult(blScalar) + resGeth := (new(gnark.G2Affine)).ScalarMultiplication(gp, scalar) + + if !bytes.Equal(resGeth.Marshal(), resBl.Serialize()) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + return 1 +} + +func fuzzCrossG2MultiExp(data []byte) int { + var ( + input = bytes.NewReader(data) + gnarkScalars []fr.Element + gnarkPoints []gnark.G2Affine + blstScalars []*blst.Scalar + blstPoints []*blst.P2Affine + ) + // n random scalars (max 17) + for i := 0; i < 17; i++ { + // note that geth/crypto/bls12381 works only with scalars <= 32bytes + s, err := randomScalar(input, fr.Modulus()) + if err != nil { + break + } + // get a random G1 point as basis + cp1, bl1, err := getG2Points(input) + if err != nil { + break + } + + gnarkScalar := new(fr.Element).SetBigInt(s) + gnarkScalars = append(gnarkScalars, *gnarkScalar) + gnarkPoints = append(gnarkPoints, *cp1) + + blstScalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + blstScalars = append(blstScalars, blstScalar) + blstPoints = append(blstPoints, bl1) + } + + if len(gnarkScalars) == 0 || len(gnarkScalars) != len(gnarkPoints) { + return 0 + } + + // gnark multi exp + cp := new(gnark.G2Affine) + cp.MultiExp(gnarkPoints, gnarkScalars, ecc.MultiExpConfig{}) + + expectedGnark := multiExpG2Gnark(gnarkPoints, gnarkScalars) + if !bytes.Equal(cp.Marshal(), expectedGnark.Marshal()) { + panic("g1 multi exponentiation mismatch") + } + + // blst multi exp + expectedBlst := blst.P2AffinesMult(blstPoints, blstScalars, 256).ToAffine() + if !bytes.Equal(cp.Marshal(), expectedBlst.Serialize()) { + panic("g1 multi exponentiation mismatch, gnark/blst") + } + return 1 +} + +func getG1Points(input io.Reader) (*gnark.G1Affine, *blst.P1Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, nil, err + return nil, nil, err } // compute a random point @@ -281,18 +339,6 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1A cp.ScalarMultiplication(&g1Gen, s) cpBytes := cp.Marshal() - // marshal gnark point -> geth point - g1 := bls12381.NewG1() - kp, err := g1.FromBytes(cpBytes) - if err != nil { - panic(fmt.Sprintf("Could not marshal gnark.G1 -> geth.G1: %v", err)) - } - - gnarkRes := g1.ToBytes(kp) - if !bytes.Equal(gnarkRes, cpBytes) { - panic(fmt.Sprintf("bytes(gnark.G1) != bytes(geth.G1)\ngnark.G1: %x\ngeth.G1: %x\n", gnarkRes, cpBytes)) - } - // marshal gnark point -> blst point scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) p1 := new(blst.P1Affine).From(scalar) @@ -301,43 +347,31 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1A panic(fmt.Sprintf("bytes(blst.G1) != bytes(geth.G1)\nblst.G1: %x\ngeth.G1: %x\n", blstRes, cpBytes)) } - return kp, cp, p1, nil + return cp, p1, nil } -func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, *blst.P2Affine, error) { +func getG2Points(input io.Reader) (*gnark.G2Affine, *blst.P2Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, nil, err + return nil, nil, err } // compute a random point - cp := new(gnark.G2Affine) + gp := new(gnark.G2Affine) _, _, _, g2Gen := gnark.Generators() - cp.ScalarMultiplication(&g2Gen, s) - cpBytes := cp.Marshal() - - // marshal gnark point -> geth point - g2 := bls12381.NewG2() - kp, err := g2.FromBytes(cpBytes) - if err != nil { - panic(fmt.Sprintf("Could not marshal gnark.G2 -> geth.G2: %v", err)) - } - - gnarkRes := g2.ToBytes(kp) - if !bytes.Equal(gnarkRes, cpBytes) { - panic(fmt.Sprintf("bytes(gnark.G2) != bytes(geth.G2)\ngnark.G2: %x\ngeth.G2: %x\n", gnarkRes, cpBytes)) - } + gp.ScalarMultiplication(&g2Gen, s) + cpBytes := gp.Marshal() // marshal gnark point -> blst point // Left pad the scalar to 32 bytes scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) p2 := new(blst.P2Affine).From(scalar) if !bytes.Equal(p2.Serialize(), cpBytes) { - panic("bytes(blst.G2) != bytes(geth.G2)") + panic("bytes(blst.G2) != bytes(bls12381.G2)") } - return kp, cp, p2, nil + return gp, p2, nil } func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { @@ -348,3 +382,29 @@ func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { } } } + +// multiExpG1Gnark is a naive implementation of G1 multi-exponentiation +func multiExpG1Gnark(gs []gnark.G1Affine, scalars []fr.Element) gnark.G1Affine { + res := gnark.G1Affine{} + for i := 0; i < len(gs); i++ { + tmp := new(gnark.G1Affine) + sb := scalars[i].Bytes() + scalarBytes := new(big.Int).SetBytes(sb[:]) + tmp.ScalarMultiplication(&gs[i], scalarBytes) + res.Add(&res, tmp) + } + return res +} + +// multiExpG1Gnark is a naive implementation of G1 multi-exponentiation +func multiExpG2Gnark(gs []gnark.G2Affine, scalars []fr.Element) gnark.G2Affine { + res := gnark.G2Affine{} + for i := 0; i < len(gs); i++ { + tmp := new(gnark.G2Affine) + sb := scalars[i].Bytes() + scalarBytes := new(big.Int).SetBytes(sb[:]) + tmp.ScalarMultiplication(&gs[i], scalarBytes) + res.Add(&res, tmp) + } + return res +} diff --git a/tests/fuzzers/bls12381/bls12381_test.go b/tests/fuzzers/bls12381/bls12381_test.go index fd782f7813..d4e5e20e04 100644 --- a/tests/fuzzers/bls12381/bls12381_test.go +++ b/tests/fuzzers/bls12381/bls12381_test.go @@ -27,6 +27,12 @@ func FuzzCrossPairing(f *testing.F) { }) } +func FuzzCrossG2MultiExp(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzzCrossG2MultiExp(data) + }) +} + func FuzzCrossG1Add(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { fuzzCrossG1Add(data) @@ -51,9 +57,9 @@ func FuzzG1Add(f *testing.F) { }) } -func FuzzG1Mul(f *testing.F) { +func FuzzCrossG1Mul(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - fuzz(blsG1Mul, data) + fuzzCrossG1Mul(data) }) } @@ -69,9 +75,9 @@ func FuzzG2Add(f *testing.F) { }) } -func FuzzG2Mul(f *testing.F) { +func FuzzCrossG2Mul(f *testing.F) { f.Fuzz(func(t *testing.T, data []byte) { - fuzz(blsG2Mul, data) + fuzzCrossG2Mul(data) }) } @@ -110,3 +116,15 @@ func FuzzG2SubgroupChecks(f *testing.F) { fuzzG2SubgroupChecks(data) }) } + +func FuzzG2Mul(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzz(blsG2Mul, data) + }) +} + +func FuzzG1Mul(f *testing.F) { + f.Fuzz(func(t *testing.T, data []byte) { + fuzz(blsG1Mul, data) + }) +} diff --git a/tests/state_test.go b/tests/state_test.go index 76d5a601c7..7b82b05e58 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -302,7 +302,8 @@ func runBenchmark(b *testing.B, t *StateTest) { context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - evm := vm.NewEVM(context, txContext, state.StateDB, config, vmconfig) + evm := vm.NewEVM(context, state.StateDB, config, vmconfig) + evm.SetTxContext(txContext) // Create "contract" for sender to cache code analysis. sender := vm.NewContract(vm.AccountRef(msg.From), vm.AccountRef(msg.From), diff --git a/tests/state_test_util.go b/tests/state_test_util.go index af2cb63d94..446ffb40d5 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -293,7 +293,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh if config.IsCancun(new(big.Int), block.Time()) && t.json.Env.ExcessBlobGas != nil { context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) } - evm := vm.NewEVM(context, txContext, st.StateDB, config, vmconfig) + evm := vm.NewEVM(context, st.StateDB, config, vmconfig) + evm.SetTxContext(txContext) if tracer := vmconfig.Tracer; tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), nil, msg.From) diff --git a/trie/proof.go b/trie/proof.go index a39d6b4ea3..40836fba33 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -48,7 +48,7 @@ func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { // The trie doesn't contain the key. tn = nil } else { @@ -371,7 +371,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error } return unset(cld, cld.Children[key[pos]], key, pos+1, removeLeft) case *shortNode: - if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { + if !bytes.HasPrefix(key[pos:], cld.Key) { // Find the fork point, it's a non-existent branch. if removeLeft { if bytes.Compare(cld.Key, key[pos:]) < 0 { @@ -434,7 +434,7 @@ func hasRightElement(node node, key []byte) bool { } node, pos = rn.Children[key[pos]], pos+1 case *shortNode: - if len(key)-pos < len(rn.Key) || !bytes.Equal(rn.Key, key[pos:pos+len(rn.Key)]) { + if !bytes.HasPrefix(key[pos:], rn.Key) { return bytes.Compare(rn.Key, key[pos:]) > 0 } node, pos = rn.Val, pos+len(rn.Key) @@ -589,7 +589,7 @@ func get(tn node, key []byte, skipResolved bool) ([]byte, node) { for { switch n := tn.(type) { case *shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if !bytes.HasPrefix(key, n.Key) { return nil, nil } tn = n.Val diff --git a/trie/trie.go b/trie/trie.go index e3f3f39248..ae2a7b21a2 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -163,7 +163,7 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no case valueNode: return n, n, false, nil case *shortNode: - if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(key[pos:], n.Key) { // key not found in trie return nil, n, false, nil } @@ -219,9 +219,6 @@ func (t *Trie) GetNode(path []byte) ([]byte, int, error) { if resolved > 0 { t.root = newroot } - if item == nil { - return nil, resolved, nil - } return item, resolved, nil } @@ -254,7 +251,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod return nil, nil, 0, nil case *shortNode: - if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) { + if !bytes.HasPrefix(path[pos:], n.Key) { // Path branches off from short node return nil, n, 0, nil } diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go index 4294f5e4be..b785e512b1 100644 --- a/trie/utils/verkle.go +++ b/trie/utils/verkle.go @@ -217,7 +217,7 @@ func StorageIndex(storageKey []byte) (*uint256.Int, byte) { // The first MAIN_STORAGE_OFFSET group will see its // first 64 slots unreachable. This is either a typo in the // spec or intended to conserve the 256-u256 - // aligment. If we decide to ever access these 64 + // alignment. If we decide to ever access these 64 // slots, uncomment this. // // Get the new offset since we now know that we are above 64. // pos.Sub(&pos, codeStorageDelta) diff --git a/triedb/states.go b/triedb/states.go index 1f9a0de522..0b03f2b9f3 100644 --- a/triedb/states.go +++ b/triedb/states.go @@ -23,7 +23,6 @@ import ( // StateSet represents a collection of mutated states during a state transition. type StateSet struct { - Destructs map[common.Hash]struct{} // Destructed accounts Accounts map[common.Hash][]byte // Mutated accounts in 'slim RLP' encoding AccountsOrigin map[common.Address][]byte // Original values of mutated accounts in 'slim RLP' encoding Storages map[common.Hash]map[common.Hash][]byte // Mutated storage slots in 'prefix-zero-trimmed' RLP format @@ -33,7 +32,6 @@ type StateSet struct { // NewStateSet initializes an empty state set. func NewStateSet() *StateSet { return &StateSet{ - Destructs: make(map[common.Hash]struct{}), Accounts: make(map[common.Hash][]byte), AccountsOrigin: make(map[common.Address][]byte), Storages: make(map[common.Hash]map[common.Hash][]byte), diff --git a/version/version.go b/version/version.go index cbd59f3e9a..52dfa0281e 100644 --- a/version/version.go +++ b/version/version.go @@ -19,6 +19,6 @@ package version const ( Major = 1 // Major version component of the current release Minor = 14 // Minor version component of the current release - Patch = 12 // Patch version component of the current release + Patch = 13 // Patch version component of the current release Meta = "unstable" // Version metadata to append to the version string )