core/vm: optimize stack and memory copy paths

core/vm/instructions.go

@@ -299,7 +299,7 @@ func opCallDataCopy(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 	// These values are checked for overflow during gas cost calculation
 	memOffset64 := memOffset.Uint64()
 	length64 := length.Uint64()
-	scope.Memory.Set(memOffset64, length64, getData(scope.Contract.Input, dataOffset64, length64))
+	scope.Memory.SetFromData(memOffset64, length64, dataOffset64, scope.Contract.Input)
 
 	return nil, nil
 }
@@ -372,8 +372,8 @@ func opExtCodeCopy(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 	}
 	addr := common.Address(a.Bytes20())
 	code := evm.StateDB.GetCode(addr)
-	codeCopy := getData(code, uint64CodeOffset, length.Uint64())
-	scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
+
+	scope.Memory.SetFromData(memOffset.Uint64(), length.Uint64(), uint64CodeOffset, code)
 
 	return nil, nil
 }
@@ -526,7 +526,7 @@ func opSstore(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 }
 
 func opJump(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	if evm.abort.Load() {
+	if evm.Cancelled() {
 		return nil, errStopToken
 	}
 	pos := scope.Stack.pop()
@@ -538,7 +538,7 @@ func opJump(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 }
 
 func opJumpi(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	if evm.abort.Load() {
+	if evm.Cancelled() {
 		return nil, errStopToken
 	}
 	pos, cond := scope.Stack.pop(), scope.Stack.pop()
@@ -571,82 +571,82 @@ func opGas(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 }
 
 func opSwap1(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap1()
+	scope.Stack.swap(1)
 	return nil, nil
 }
 
 func opSwap2(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap2()
+	scope.Stack.swap(2)
 	return nil, nil
 }
 
 func opSwap3(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap3()
+	scope.Stack.swap(3)
 	return nil, nil
 }
 
 func opSwap4(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap4()
+	scope.Stack.swap(4)
 	return nil, nil
 }
 
 func opSwap5(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap5()
+	scope.Stack.swap(5)
 	return nil, nil
 }
 
 func opSwap6(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap6()
+	scope.Stack.swap(6)
 	return nil, nil
 }
 
 func opSwap7(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap7()
+	scope.Stack.swap(7)
 	return nil, nil
 }
 
 func opSwap8(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap8()
+	scope.Stack.swap(8)
 	return nil, nil
 }
 
 func opSwap9(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap9()
+	scope.Stack.swap(9)
 	return nil, nil
 }
 
 func opSwap10(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap10()
+	scope.Stack.swap(10)
 	return nil, nil
 }
 
 func opSwap11(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap11()
+	scope.Stack.swap(11)
 	return nil, nil
 }
 
 func opSwap12(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap12()
+	scope.Stack.swap(12)
 	return nil, nil
 }
 
 func opSwap13(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap13()
+	scope.Stack.swap(13)
 	return nil, nil
 }
 
 func opSwap14(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap14()
+	scope.Stack.swap(14)
 	return nil, nil
 }
 
 func opSwap15(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap15()
+	scope.Stack.swap(15)
 	return nil, nil
 }
 
 func opSwap16(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.swap16()
+	scope.Stack.swap(16)
 	return nil, nil
 }
 

core/vm/instructions_test.go

@@ -109,8 +109,8 @@ func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFu
 		stack.push(x)
 		stack.push(y)
 		opFn(&pc, evm, &ScopeContext{nil, stack, nil})
-		if len(stack.data) != 1 {
-			t.Errorf("Expected one item on stack after %v, got %d: ", name, len(stack.data))
+		if stack.top != 1 {
+			t.Errorf("Expected one item on stack after %v, got %d: ", name, stack.top)
 		}
 		actual := stack.pop()
 
@@ -705,8 +705,8 @@ func TestRandom(t *testing.T) {
 			pc    = uint64(0)
 		)
 		opRandom(&pc, evm, &ScopeContext{nil, stack, nil})
-		if len(stack.data) != 1 {
-			t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data))
+		if stack.top != 1 {
+			t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, stack.top)
 		}
 		actual := stack.pop()
 		expected, overflow := uint256.FromBig(new(big.Int).SetBytes(tt.random.Bytes()))
@@ -747,8 +747,8 @@ func TestBlobHash(t *testing.T) {
 		evm.SetTxContext(TxContext{BlobHashes: tt.hashes})
 		stack.push(uint256.NewInt(tt.idx))
 		opBlobHash(&pc, evm, &ScopeContext{nil, stack, nil})
-		if len(stack.data) != 1 {
-			t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, len(stack.data))
+		if stack.top != 1 {
+			t.Errorf("Expected one item on stack after %v, got %d: ", tt.name, stack.top)
 		}
 		actual := stack.pop()
 		expected, overflow := uint256.FromBig(new(big.Int).SetBytes(tt.expect.Bytes()))

core/vm/memory.go

@@ -65,6 +65,32 @@ func (m *Memory) Set(offset, size uint64, value []byte) {
 	}
 }
 
+// SetFromData copies src[dataOffset:] into m.store[memOffset:memOffset+size],
+// zero-padding if src is shorter than size. This avoids allocating an intermediate
+// padded buffer (unlike getData + Set).
+func (m *Memory) SetFromData(memOffset, size, dataOffset uint64, data []byte) {
+	if size == 0 {
+		return
+	}
+	if memOffset+size > uint64(len(m.store)) {
+		panic("invalid memory: store empty")
+	}
+	dst := m.store[memOffset : memOffset+size]
+	dataLen := uint64(len(data))
+	if dataOffset >= dataLen {
+		// All zeros.
+		clear(dst)
+		return
+	}
+	avail := dataLen - dataOffset
+	if avail >= size {
+		copy(dst, data[dataOffset:dataOffset+size])
+	} else {
+		copy(dst[:avail], data[dataOffset:dataOffset+avail])
+		clear(dst[avail:])
+	}
+}
+
 // Set32 sets the 32 bytes starting at offset to the value of val, left-padded with zeroes to
 // 32 bytes.
 func (m *Memory) Set32(offset uint64, val *uint256.Int) {

core/vm/stack.go

@@ -22,9 +22,11 @@ import (
 	"github.com/holiman/uint256"
 )
 
+const stackLimit = 1024
+
 var stackPool = sync.Pool{
-	New: func() interface{} {
-		return &Stack{data: make([]uint256.Int, 0, 16)}
+	New: func() any {
+		return &Stack{}
 	},
 }
 
@@ -32,7 +34,8 @@ var stackPool = sync.Pool{
 // expected to be changed and modified. stack does not take care of adding newly
 // initialized objects.
 type Stack struct {
-	data []uint256.Int
+	data [stackLimit]uint256.Int
+	top  int
 }
 
 func newstack() *Stack {
@@ -40,88 +43,45 @@ func newstack() *Stack {
 }
 
 func returnStack(s *Stack) {
-	s.data = s.data[:0]
+	s.top = 0
 	stackPool.Put(s)
 }
 
 // Data returns the underlying uint256.Int array.
 func (st *Stack) Data() []uint256.Int {
-	return st.data
+	return st.data[:st.top]
 }
 
 func (st *Stack) push(d *uint256.Int) {
 	// NOTE push limit (1024) is checked in baseCheck
-	st.data = append(st.data, *d)
+	st.data[st.top] = *d
+	st.top++
 }
 
 func (st *Stack) pop() (ret uint256.Int) {
-	ret = st.data[len(st.data)-1]
-	st.data = st.data[:len(st.data)-1]
+	st.top--
+	ret = st.data[st.top]
 	return
 }
 
 func (st *Stack) len() int {
-	return len(st.data)
+	return st.top
 }
 
-func (st *Stack) swap1() {
-	st.data[st.len()-2], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-2]
-}
-func (st *Stack) swap2() {
-	st.data[st.len()-3], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-3]
-}
-func (st *Stack) swap3() {
-	st.data[st.len()-4], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-4]
-}
-func (st *Stack) swap4() {
-	st.data[st.len()-5], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-5]
-}
-func (st *Stack) swap5() {
-	st.data[st.len()-6], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-6]
-}
-func (st *Stack) swap6() {
-	st.data[st.len()-7], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-7]
-}
-func (st *Stack) swap7() {
-	st.data[st.len()-8], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-8]
-}
-func (st *Stack) swap8() {
-	st.data[st.len()-9], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-9]
-}
-func (st *Stack) swap9() {
-	st.data[st.len()-10], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-10]
-}
-func (st *Stack) swap10() {
-	st.data[st.len()-11], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-11]
-}
-func (st *Stack) swap11() {
-	st.data[st.len()-12], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-12]
-}
-func (st *Stack) swap12() {
-	st.data[st.len()-13], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-13]
-}
-func (st *Stack) swap13() {
-	st.data[st.len()-14], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-14]
-}
-func (st *Stack) swap14() {
-	st.data[st.len()-15], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-15]
-}
-func (st *Stack) swap15() {
-	st.data[st.len()-16], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-16]
-}
-func (st *Stack) swap16() {
-	st.data[st.len()-17], st.data[st.len()-1] = st.data[st.len()-1], st.data[st.len()-17]
+func (st *Stack) swap(n int) {
+	st.data[st.top-n-1], st.data[st.top-1] = st.data[st.top-1], st.data[st.top-n-1]
 }
 
 func (st *Stack) dup(n int) {
-	st.push(&st.data[st.len()-n])
+	st.data[st.top] = st.data[st.top-n]
+	st.top++
 }
 
 func (st *Stack) peek() *uint256.Int {
-	return &st.data[st.len()-1]
+	return &st.data[st.top-1]
 }
 
 // Back returns the n'th item in stack
 func (st *Stack) Back(n int) *uint256.Int {
-	return &st.data[st.len()-n-1]
+	return &st.data[st.top-n-1]
 }