From f3f80f222d6174240ab894d33f281bf6d355215b Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:57:28 +0200 Subject: [PATCH] core/vm: introduce arena-based stack with instruction rework Squashed from 11 commits on bigstack-rebased: - use methods over direct access to stack internals - fix benchmark - speed up stack - speed up push1, push2 (x2) - reuse stack arena - rework more instructions - use arena for all instructions - add defensive stack pool return - plus lint and compile fixes Co-authored-by: Martin Holst Swende Co-authored-by: Marius van der Wijden --- core/state_prefetcher.go | 1 + core/state_processor.go | 1 + core/vm/eips.go | 28 ++--- core/vm/evm.go | 9 ++ core/vm/gas_table.go | 4 +- core/vm/instructions.go | 72 ++++++----- core/vm/instructions_test.go | 77 +++++++----- core/vm/interpreter.go | 6 +- core/vm/interpreter_test.go | 2 +- core/vm/runtime/runtime_test.go | 60 ++++++++++ core/vm/stack.go | 203 ++++++++++++++++++++------------ 11 files changed, 299 insertions(+), 164 deletions(-) diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index c91d40d94f..008108b32e 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -93,6 +93,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c } // Execute the message to preload the implicit touched states evm := vm.NewEVM(NewEVMBlockContext(header, p.chain, nil), stateCpy, p.config, cfg) + defer evm.Free() // Convert the transaction into an executable message and pre-cache its sender msg, err := TransactionToMessage(tx, signer, header.BaseFee) diff --git a/core/state_processor.go b/core/state_processor.go index fda3bf8fe7..2226038cb7 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -88,6 +88,7 @@ func (p *StateProcessor) Process(ctx context.Context, block *types.Block, stated // Apply pre-execution system calls. context = NewEVMBlockContext(header, p.chain, nil) evm := vm.NewEVM(context, tracingStateDB, config, cfg) + defer evm.Free() if beaconRoot := block.BeaconRoot(); beaconRoot != nil { ProcessBeaconBlockRoot(*beaconRoot, evm) diff --git a/core/vm/eips.go b/core/vm/eips.go index 54e5cb0c60..33af8fd4fd 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" ) var activators = map[int]func(*JumpTable){ @@ -92,8 +91,7 @@ func enable1884(jt *JumpTable) { } func opSelfBalance(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - balance := evm.StateDB.GetBalance(scope.Contract.Address()) - scope.Stack.push(balance) + scope.Stack.get().Set(evm.StateDB.GetBalance(scope.Contract.Address())) return nil, nil } @@ -111,8 +109,7 @@ func enable1344(jt *JumpTable) { // opChainID implements CHAINID opcode func opChainID(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - chainId, _ := uint256.FromBig(evm.chainConfig.ChainID) - scope.Stack.push(chainId) + scope.Stack.get().SetFromBig(evm.chainConfig.ChainID) return nil, nil } @@ -222,8 +219,7 @@ func opTstore(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { // opBaseFee implements BASEFEE opcode func opBaseFee(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - baseFee, _ := uint256.FromBig(evm.Context.BaseFee) - scope.Stack.push(baseFee) + scope.Stack.get().SetFromBig(evm.Context.BaseFee) return nil, nil } @@ -240,7 +236,7 @@ func enable3855(jt *JumpTable) { // opPush0 implements the PUSH0 opcode func opPush0(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int)) + scope.Stack.get().Clear() return nil, nil } @@ -291,8 +287,7 @@ func opBlobHash(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { // opBlobBaseFee implements BLOBBASEFEE opcode func opBlobBaseFee(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - blobBaseFee, _ := uint256.FromBig(evm.Context.BlobBaseFee) - scope.Stack.push(blobBaseFee) + scope.Stack.get().SetFromBig(evm.Context.BlobBaseFee) return nil, nil } @@ -397,11 +392,11 @@ func opExtCodeCopyEIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, er func opPush1EIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( codeLen = uint64(len(scope.Contract.Code)) - integer = new(uint256.Int) + elem = scope.Stack.get() ) *pc += 1 if *pc < codeLen { - scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc]))) + elem.SetUint64(uint64(scope.Contract.Code[*pc])) if !scope.Contract.IsDeployment && !scope.Contract.IsSystemCall && *pc%31 == 0 { // touch next chunk if PUSH1 is at the boundary. if so, *pc has @@ -414,7 +409,7 @@ func opPush1EIP4762(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } } } else { - scope.Stack.push(integer.Clear()) + elem.Clear() } return nil, nil } @@ -426,12 +421,11 @@ func makePushEIP4762(size uint64, pushByteSize int) executionFunc { start = min(codeLen, int(*pc+1)) end = min(codeLen, start+pushByteSize) ) - scope.Stack.push(new(uint256.Int).SetBytes( + scope.Stack.get().SetBytes( common.RightPadBytes( scope.Contract.Code[start:end], pushByteSize, - )), - ) + )) if !scope.Contract.IsDeployment && !scope.Contract.IsSystemCall { contractAddr := scope.Contract.Address() @@ -583,7 +577,7 @@ func enable7702(jt *JumpTable) { // opSlotNum enables the SLOTNUM opcode func opSlotNum(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(uint256.NewInt(evm.Context.SlotNum)) + scope.Stack.get().SetUint64(evm.Context.SlotNum) return nil, nil } diff --git a/core/vm/evm.go b/core/vm/evm.go index 59e301c0a7..6fd4d96377 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -127,6 +127,8 @@ type EVM struct { readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse + + arena *stackArena } // NewEVM constructs an EVM instance with the supplied block context, state @@ -141,6 +143,7 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon chainConfig: chainConfig, chainRules: chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time), jumpDests: newMapJumpDests(), + arena: newArena(), } evm.precompiles = activePrecompiledContracts(evm.chainRules) @@ -223,6 +226,12 @@ func (evm *EVM) Cancel() { evm.abort.Store(true) } +// Free returns some memory allocated by the EVM, should be called after the EVM was used +// for the last time. Not necessary, but an improvement. +func (evm *EVM) Free() { + returnStack(evm.arena) +} + // Cancelled returns true if Cancel has been called func (evm *EVM) Cancelled() bool { return evm.abort.Load() diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index b3259b2ec7..de47e07ac0 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -352,7 +352,7 @@ func gasCreate2Eip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, } func gasExpFrontier(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (GasCosts, error) { - expByteLen := uint64((stack.data[stack.len()-2].BitLen() + 7) / 8) + expByteLen := uint64((stack.Back(1).BitLen() + 7) / 8) var ( gas = expByteLen * params.ExpByteFrontier // no overflow check required. Max is 256 * ExpByte gas @@ -365,7 +365,7 @@ func gasExpFrontier(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem } func gasExpEIP158(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (GasCosts, error) { - expByteLen := uint64((stack.data[stack.len()-2].BitLen() + 7) / 8) + expByteLen := uint64((stack.Back(1).BitLen() + 7) / 8) var ( gas = expByteLen * params.ExpByteEIP158 // no overflow check required. Max is 256 * ExpByte gas diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 3311af0d22..9a84726349 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" - "github.com/holiman/uint256" ) func opAdd(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { @@ -244,7 +243,7 @@ func opKeccak256(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opAddress(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Address().Bytes())) + scope.Stack.get().SetBytes(scope.Contract.Address().Bytes()) return nil, nil } @@ -256,17 +255,17 @@ func opBalance(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opOrigin(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetBytes(evm.Origin.Bytes())) + scope.Stack.get().SetBytes(evm.Origin.Bytes()) return nil, nil } func opCaller(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Caller().Bytes())) + scope.Stack.get().SetBytes(scope.Contract.Caller().Bytes()) return nil, nil } func opCallValue(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(scope.Contract.value) + scope.Stack.get().Set(scope.Contract.value) return nil, nil } @@ -282,7 +281,7 @@ func opCallDataLoad(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opCallDataSize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Input)))) + scope.Stack.get().SetUint64(uint64(len(scope.Contract.Input))) return nil, nil } @@ -305,7 +304,7 @@ func opCallDataCopy(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opReturnDataSize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(evm.returnData)))) + scope.Stack.get().SetUint64(uint64(len(evm.returnData))) return nil, nil } @@ -338,7 +337,7 @@ func opExtCodeSize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opCodeSize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(uint64(len(scope.Contract.Code)))) + scope.Stack.get().SetUint64(uint64(len(scope.Contract.Code))) return nil, nil } @@ -416,7 +415,7 @@ func opExtCodeHash(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opGasprice(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(evm.GasPrice.Clone()) + scope.Stack.get().Set(evm.GasPrice) return nil, nil } @@ -451,35 +450,32 @@ func opBlockhash(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opCoinbase(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetBytes(evm.Context.Coinbase.Bytes())) + scope.Stack.get().SetBytes(evm.Context.Coinbase.Bytes()) return nil, nil } func opTimestamp(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(evm.Context.Time)) + scope.Stack.get().SetUint64(evm.Context.Time) return nil, nil } func opNumber(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - v, _ := uint256.FromBig(evm.Context.BlockNumber) - scope.Stack.push(v) + scope.Stack.get().SetFromBig(evm.Context.BlockNumber) return nil, nil } func opDifficulty(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - v, _ := uint256.FromBig(evm.Context.Difficulty) - scope.Stack.push(v) + scope.Stack.get().SetFromBig(evm.Context.Difficulty) return nil, nil } func opRandom(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - v := new(uint256.Int).SetBytes(evm.Context.Random.Bytes()) - scope.Stack.push(v) + scope.Stack.get().SetBytes(evm.Context.Random.Bytes()) return nil, nil } func opGasLimit(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(evm.Context.GasLimit)) + scope.Stack.get().SetUint64(evm.Context.GasLimit) return nil, nil } @@ -556,17 +552,17 @@ func opJumpdest(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } func opPc(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(*pc)) + scope.Stack.get().SetUint64(*pc) return nil, nil } func opMsize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(uint64(scope.Memory.Len()))) + scope.Stack.get().SetUint64(uint64(scope.Memory.Len())) return nil, nil } func opGas(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { - scope.Stack.push(new(uint256.Int).SetUint64(scope.Contract.Gas.RegularGas)) + scope.Stack.get().SetUint64(scope.Contract.Gas.RegularGas) return nil, nil } @@ -1034,10 +1030,10 @@ func opSwapN(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: n + 1} } - // The (n+1)‘th stack item is swapped with the top of the stack. - indexTop := scope.Stack.len() - 1 - indexN := scope.Stack.len() - 1 - n - scope.Stack.data[indexTop], scope.Stack.data[indexN] = scope.Stack.data[indexN], scope.Stack.data[indexTop] + // The (n+1)’th stack item is swapped with the top of the stack. + top := scope.Stack.peek() + nth := scope.Stack.Back(n) + *top, *nth = *nth, *top *pc += 1 return nil, nil } @@ -1067,10 +1063,10 @@ func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { return nil, &ErrStackUnderflow{stackLen: scope.Stack.len(), required: need} } - // The (n+1)‘th stack item is swapped with the (m+1)‘th stack item. - indexN := scope.Stack.len() - 1 - n - indexM := scope.Stack.len() - 1 - m - scope.Stack.data[indexN], scope.Stack.data[indexM] = scope.Stack.data[indexM], scope.Stack.data[indexN] + // The (n+1)’th stack item is swapped with the (m+1)’th stack item. + nth := scope.Stack.Back(n) + mth := scope.Stack.Back(m) + *nth, *mth = *mth, *nth *pc += 1 return nil, nil } @@ -1106,13 +1102,13 @@ func makeLog(size int) executionFunc { func opPush1(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( codeLen = uint64(len(scope.Contract.Code)) - integer = new(uint256.Int) + elem = scope.Stack.get() ) *pc += 1 if *pc < codeLen { - scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc]))) + elem.SetUint64(uint64(scope.Contract.Code[*pc])) } else { - scope.Stack.push(integer.Clear()) + elem.Clear() } return nil, nil } @@ -1121,14 +1117,14 @@ func opPush1(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { func opPush2(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { var ( codeLen = uint64(len(scope.Contract.Code)) - integer = new(uint256.Int) + elem = scope.Stack.get() ) if *pc+2 < codeLen { - scope.Stack.push(integer.SetBytes2(scope.Contract.Code[*pc+1 : *pc+3])) + elem.SetBytes2(scope.Contract.Code[*pc+1 : *pc+3]) } else if *pc+1 < codeLen { - scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc+1]) << 8)) + elem.SetUint64(uint64(scope.Contract.Code[*pc+1]) << 8) } else { - scope.Stack.push(integer.Clear()) + elem.Clear() } *pc += 2 return nil, nil @@ -1142,13 +1138,13 @@ func makePush(size uint64, pushByteSize int) executionFunc { start = min(codeLen, int(*pc+1)) end = min(codeLen, start+pushByteSize) ) - a := new(uint256.Int).SetBytes(scope.Contract.Code[start:end]) + a := scope.Stack.get() + a.SetBytes(scope.Contract.Code[start:end]) // Missing bytes: pushByteSize - len(pushData) if missing := pushByteSize - (end - start); missing > 0 { a.Lsh(a, uint(8*missing)) } - scope.Stack.push(a) *pc += size return nil, nil } diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index dbe055f2ac..354d2ce5ab 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -25,6 +25,7 @@ import ( "os" "strings" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" @@ -98,7 +99,7 @@ func init() { func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) @@ -109,8 +110,8 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu stack.push(x) stack.push(y) opFn(&pc, evm, &ScopeContext{nil, stack, nil}) - if len(stack.data) != 1 { - t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data)) + if stack.len() != 1 { + t.Errorf("Expected one item on stack after %v, got %d: ", name, stack.len()) } actual := stack.pop() @@ -196,7 +197,7 @@ func TestSAR(t *testing.T) { func TestAddMod(t *testing.T) { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) tests := []struct { @@ -239,7 +240,7 @@ func TestWriteExpectedValues(t *testing.T) { getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) result := make([]TwoOperandTestcase, len(args)) @@ -282,23 +283,40 @@ func TestJsonTestcases(t *testing.T) { func opBenchmark(bench *testing.B, op executionFunc, args ...string) { var ( - evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - scope = &ScopeContext{nil, stack, nil} + evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) + stack = newStackForTesting() + code = []byte{} + opPush32 = makePush(32, 32) ) // convert args intArgs := make([]*uint256.Int, len(args)) for i, arg := range args { + code = append(code, common.LeftPadBytes(common.Hex2Bytes(arg), 32)...) intArgs[i] = new(uint256.Int).SetBytes(common.Hex2Bytes(arg)) } pc := uint64(0) - for bench.Loop() { - for _, arg := range intArgs { - stack.push(arg) + scope := &ScopeContext{nil, stack, &Contract{Code: code}} + start := time.Now() + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + for range len(args) { + opPush32(&pc, evm, scope) + pc += 32 } op(&pc, evm, scope) - stack.pop() + opPop(&pc, evm, scope) } + bench.StopTimer() + elapsed := uint64(time.Since(start)) + if elapsed < 1 { + elapsed = 1 + } + reqGas := uint64(len(args))*GasFastestStep + GasFastestStep + GasQuickStep + gasUsed := reqGas * uint64(bench.N) + bench.ReportMetric(float64(reqGas), "gas/op") + // Keep it as uint64, multiply 100 to get two digit float later + mgasps := (100 * 1000 * gasUsed) / elapsed + bench.ReportMetric(float64(mgasps)/100, "mgas/s") for i, arg := range args { want := new(uint256.Int).SetBytes(common.Hex2Bytes(arg)) @@ -519,7 +537,7 @@ func BenchmarkOpIsZero(b *testing.B) { func TestOpMstore(t *testing.T) { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() mem = NewMemory() ) mem.Resize(64) @@ -542,7 +560,7 @@ func TestOpMstore(t *testing.T) { func BenchmarkOpMstore(bench *testing.B) { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() mem = NewMemory() ) mem.Resize(64) @@ -561,7 +579,7 @@ func TestOpTstore(t *testing.T) { var ( statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) evm = NewEVM(BlockContext{}, statedb, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() mem = NewMemory() caller = common.Address{} to = common.Address{1} @@ -600,7 +618,7 @@ func TestOpTstore(t *testing.T) { func BenchmarkOpKeccak256(bench *testing.B) { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() mem = NewMemory() ) mem.Resize(32) @@ -672,7 +690,7 @@ func TestCreate2Addresses(t *testing.T) { codeHash := crypto.Keccak256(code) address := crypto.CreateAddress2(origin, salt, codeHash) /* - stack := newstack() + stack := newStackForTesting() // salt, but we don't need that for this test stack.push(big.NewInt(int64(len(code)))) //size stack.push(big.NewInt(0)) // memstart @@ -701,12 +719,12 @@ func TestRandom(t *testing.T) { } { var ( evm = NewEVM(BlockContext{Random: &tt.random}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) opRandom(&pc, evm, &ScopeContext{nil, stack, nil}) - if len(stack.data) != 1 { - t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) + if have, want := stack.len(), 1; have != want { + t.Errorf("test '%v': want %d item(s) on stack, have %d: ", tt.name, have, want) } actual := stack.pop() expected, overflow := uint256.FromBig(new(big.Int).SetBytes(tt.random.Bytes())) @@ -741,14 +759,14 @@ func TestBlobHash(t *testing.T) { } { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) evm.SetTxContext(TxContext{BlobHashes: tt.hashes}) stack.push(uint256.NewInt(tt.idx)) opBlobHash(&pc, evm, &ScopeContext{nil, stack, nil}) - if len(stack.data) != 1 { - t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data)) + if have, want := stack.len(), 1; have != want { + t.Errorf("test '%v': want %d item(s) on stack, have %d: ", tt.name, have, want) } actual := stack.pop() expected, overflow := uint256.FromBig(new(big.Int).SetBytes(tt.expect.Bytes())) @@ -844,7 +862,7 @@ func TestOpMCopy(t *testing.T) { } { var ( evm = NewEVM(BlockContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() + stack = newStackForTesting() pc = uint64(0) ) data := common.FromHex(strings.ReplaceAll(tc.pre, " ", "")) @@ -907,7 +925,7 @@ func TestPush(t *testing.T) { scope := &ScopeContext{ Memory: nil, - Stack: newstack(), + Stack: newStackForTesting(), Contract: &Contract{ Code: code, }, @@ -988,7 +1006,7 @@ func TestOpCLZ(t *testing.T) { } for _, tc := range tests { // prepare a fresh stack and PC - stack := newstack() + stack := newStackForTesting() pc := uint64(0) // parse input @@ -1111,7 +1129,7 @@ func TestEIP8024_Execution(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { code := common.FromHex(tc.codeHex) - stack := newstack() + stack := newStackForTesting() pc := uint64(0) scope := &ScopeContext{Stack: stack, Contract: &Contract{Code: code}} var err error @@ -1189,8 +1207,9 @@ func TestEIP8024_Execution(t *testing.T) { t.Fatalf("unexpected error: %v", err) } got := make([]uint64, 0, stack.len()) - for i := stack.len() - 1; i >= 0; i-- { - got = append(got, stack.data[i].Uint64()) + data := stack.Data() + for i := len(data) - 1; i >= 0; i-- { + got = append(got, data[i].Uint64()) } if len(got) != len(tc.wantVals) { t.Fatalf("stack len=%d; want %d", len(got), len(tc.wantVals)) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 9dc0a0b787..4c278fc857 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -116,8 +116,8 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte var ( op OpCode // current opcode jumpTable *JumpTable = evm.table - mem = NewMemory() // bound memory - stack = newstack() // local stack + mem = NewMemory() // bound memory + stack = evm.arena.stack() // local stack callContext = &ScopeContext{ Memory: mem, Stack: stack, @@ -140,7 +140,7 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte // so that it gets executed _after_: the OnOpcode needs the stacks before // they are returned to the pools defer func() { - returnStack(stack) + stack.release() mem.Free() }() contract.Input = input diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index 69c2316907..868cb12d04 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -83,7 +83,7 @@ func BenchmarkInterpreter(b *testing.B) { evm = NewEVM(BlockContext{BlockNumber: big.NewInt(1), Time: 1, Random: &common.Hash{}}, statedb, params.MergedTestChainConfig, Config{}) startGas uint64 = 100_000_000 value = uint256.NewInt(0) - stack = newstack() + stack = newStackForTesting() mem = NewMemory() contract = NewContract(common.Address{}, common.Address{}, value, NewGasBudget(startGas), nil) ) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 40fb770454..bc2ffd622d 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -959,3 +959,63 @@ func TestDelegatedAccountAccessCost(t *testing.T) { } } } + +func TestManyLargeStacks(t *testing.T) { + // This piece of code will push 512 items to the stack, and then call itself + // recursively. + code := make([]byte, 10) + for i := range code { + code[i] = byte(vm.PUSH0) + } + code = append(code, []byte{ + byte(vm.ADDRESS), // address to call + byte(vm.GAS), + byte(vm.CALL), + }...) + + main := common.HexToAddress("0xbb") + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + statedb.SetCode(main, code, tracing.CodeChangeUnspecified) + + //tracer := logger.NewJSONLogger(nil, os.Stdout) + var tracer *tracing.Hooks + _, _, err := Call(main, nil, &Config{ + GasLimit: 10_000_000, + State: statedb, + EVMConfig: vm.Config{ + Tracer: tracer, + }}) + if err != nil { + t.Fatal("didn't expect error", err) + } +} + +func BenchmarkLargeDeepStacks(b *testing.B) { + // This piece of code will push 512 items to the stack, and then call itself + // recursively. + code := make([]byte, 512) + for i := range code { + code[i] = byte(vm.PUSH0) + } + code = append(code, []byte{ + byte(vm.ADDRESS), // address to call + byte(vm.GAS), + byte(vm.CALL), + }...) + benchmarkNonModifyingCode(10_000_000, code, "deep-large-stacks-10M", "", b) +} + +func BenchmarkShortDeepStacks(b *testing.B) { + // This piece of code will push a few items to the stack, and then call itself + // recursively. + code := make([]byte, 8) + for i := range code { + code[i] = byte(vm.PUSH0) + } + code = append(code, []byte{ + byte(vm.ADDRESS), // address to call + byte(vm.GAS), + byte(vm.CALL), + }...) + benchmarkNonModifyingCode(10_000_000, code, "deep-short-stacks-10M", "", b) +} diff --git a/core/vm/stack.go b/core/vm/stack.go index 879dc9aa6d..b65b002a0d 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -17,111 +17,166 @@ package vm import ( + "slices" "sync" "github.com/holiman/uint256" ) +// stackArena is an arena which actual evm stacks use for data storage +type stackArena struct { + data []uint256.Int + top int // first free slot +} + +func newArena() *stackArena { + return stackPool.New().(*stackArena) +} + var stackPool = sync.Pool{ - New: func() interface{} { - return &Stack{data: make([]uint256.Int, 0, 16)} + New: func() any { + return &stackArena{ + data: make([]uint256.Int, 1025), + } }, } +func returnStack(arena *stackArena) { + arena.top = 0 // defensive, not strictly needed as s.inner.top = s.bottom in release() + stackPool.Put(arena) +} + +// stack returns an instance of a stack which uses the underlying arena. The instance +// must be released by invoking the (*Stack).release() method +func (sa *stackArena) stack() *Stack { + // make sure every substack has at least 1024 elements + if len(sa.data) <= sa.top+1024 { + // we need to grow the arena + sa.data = slices.Grow(sa.data, 1024) + sa.data = sa.data[:cap(sa.data)] + } + return &Stack{ + bottom: sa.top, + size: 0, + inner: sa, + } +} + +// newStackForTesting is meant to be used solely for testing. It creates a stack +// backed by a newly allocated arena. +func newStackForTesting() *Stack { + arena := &stackArena{ + data: make([]uint256.Int, 1025), + } + return arena.stack() +} + // Stack is an object for basic stack operations. Items popped to the stack are // expected to be changed and modified. stack does not take care of adding newly // initialized objects. type Stack struct { - data []uint256.Int + bottom int // bottom is the index of the first element of this stack + size int // size is the number of elements in this stack + inner *stackArena } -func newstack() *Stack { - return stackPool.Get().(*Stack) -} - -func returnStack(s *Stack) { - s.data = s.data[:0] - stackPool.Put(s) +// release un-claims the area of the arena which was claimed by the stack. +func (s *Stack) release() { + // When the stack is returned, need to notify the arena that the new 'top' is + // the returned stack's bottom. + s.inner.top = s.bottom } // Data returns the underlying uint256.Int array. -func (st *Stack) Data() []uint256.Int { - return st.data +func (s *Stack) Data() []uint256.Int { + return s.inner.data[s.bottom : s.bottom+s.size] } -func (st *Stack) push(d *uint256.Int) { - // NOTE push limit (1024) is checked in baseCheck - st.data = append(st.data, *d) +func (s *Stack) push(d *uint256.Int) { + s.inner.data[s.inner.top] = *d + s.inner.top++ + s.size++ } -func (st *Stack) pop() (ret uint256.Int) { - ret = st.data[len(st.data)-1] - st.data = st.data[:len(st.data)-1] - return +// get returns a pointer to a newly created element +// on top of the stack +func (s *Stack) get() *uint256.Int { + elem := &s.inner.data[s.inner.top] + s.inner.top++ + s.size++ + return elem } -func (st *Stack) len() int { - return len(st.data) +func (s *Stack) pop() uint256.Int { + s.inner.top-- + s.size-- + return s.inner.data[s.inner.top] } -func (st *Stack) swap1() { - st.data[st.len()-2], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-2] -} -func (st *Stack) swap2() { - st.data[st.len()-3], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-3] -} -func (st *Stack) swap3() { - st.data[st.len()-4], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-4] -} -func (st *Stack) swap4() { - st.data[st.len()-5], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-5] -} -func (st *Stack) swap5() { - st.data[st.len()-6], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-6] -} -func (st *Stack) swap6() { - st.data[st.len()-7], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-7] -} -func (st *Stack) swap7() { - st.data[st.len()-8], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-8] -} -func (st *Stack) swap8() { - st.data[st.len()-9], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-9] -} -func (st *Stack) swap9() { - st.data[st.len()-10], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-10] -} -func (st *Stack) swap10() { - st.data[st.len()-11], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-11] -} -func (st *Stack) swap11() { - st.data[st.len()-12], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-12] -} -func (st *Stack) swap12() { - st.data[st.len()-13], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-13] -} -func (st *Stack) swap13() { - st.data[st.len()-14], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-14] -} -func (st *Stack) swap14() { - st.data[st.len()-15], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-15] -} -func (st *Stack) swap15() { - st.data[st.len()-16], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-16] -} -func (st *Stack) swap16() { - st.data[st.len()-17], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-17] +func (s *Stack) len() int { + return s.size } -func (st *Stack) dup(n int) { - st.push(&st.data[st.len()-n]) +func (s *Stack) swap1() { + s.inner.data[s.bottom+s.size-2], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-2] +} +func (s *Stack) swap2() { + s.inner.data[s.bottom+s.size-3], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-3] +} +func (s *Stack) swap3() { + s.inner.data[s.bottom+s.size-4], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-4] +} +func (s *Stack) swap4() { + s.inner.data[s.bottom+s.size-5], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-5] +} +func (s *Stack) swap5() { + s.inner.data[s.bottom+s.size-6], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-6] +} +func (s *Stack) swap6() { + s.inner.data[s.bottom+s.size-7], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-7] +} +func (s *Stack) swap7() { + s.inner.data[s.bottom+s.size-8], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-8] +} +func (s *Stack) swap8() { + s.inner.data[s.bottom+s.size-9], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-9] +} +func (s *Stack) swap9() { + s.inner.data[s.bottom+s.size-10], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-10] +} +func (s *Stack) swap10() { + s.inner.data[s.bottom+s.size-11], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-11] +} +func (s *Stack) swap11() { + s.inner.data[s.bottom+s.size-12], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-12] +} +func (s *Stack) swap12() { + s.inner.data[s.bottom+s.size-13], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-13] +} +func (s *Stack) swap13() { + s.inner.data[s.bottom+s.size-14], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-14] +} +func (s *Stack) swap14() { + s.inner.data[s.bottom+s.size-15], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-15] +} +func (s *Stack) swap15() { + s.inner.data[s.bottom+s.size-16], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-16] +} +func (s *Stack) swap16() { + s.inner.data[s.bottom+s.size-17], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-17] } -func (st *Stack) peek() *uint256.Int { - return &st.data[st.len()-1] +func (s *Stack) dup(n int) { + s.inner.data[s.bottom+s.size] = s.inner.data[s.bottom+s.size-n] + s.size++ + s.inner.top++ +} + +func (s *Stack) peek() *uint256.Int { + return &s.inner.data[s.bottom+s.size-1] } // Back returns the n'th item in stack -func (st *Stack) Back(n int) *uint256.Int { - return &st.data[st.len()-n-1] +func (s *Stack) Back(n int) *uint256.Int { + return &s.inner.data[s.bottom+s.size-n-1] }