Merge 008e27ae8a into dddbaa4bf3

core/vm/eips.go

@@ -237,7 +237,8 @@ func enable3855(jt *JumpTable) {
 
 // opPush0 implements the PUSH0 opcode
 func opPush0(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.get().Clear()
+	elem := scope.Stack.get()
+	elem.Clear()
 	return nil, nil
 }
 

core/vm/evm.go

@@ -131,6 +131,11 @@ type EVM struct {
 	returnData []byte // Last CALL's return data for subsequent reuse
 
 	arena *stackArena
+
+	// forceTableLoop is a test-only switch. The differential test sets it so
+	// that Run uses the table loop (execTraced) for the whole call tree, which
+	// lets the test compare it against the generated dispatch.
+	forceTableLoop bool
 }
 
 // NewEVM constructs an EVM instance with the supplied block context, state

core/vm/gascosts.go

@@ -84,15 +84,14 @@ func (g *GasBudget) Charge(cost GasCosts) (GasBudget, bool) {
 	return prior, ok
 }
 
-// ChargeRegularOnly deducts a regular-only cost. It's always preferred for
-// performance consideration if the opcode doesn't have any state cost.
-func (g *GasBudget) ChargeRegularOnly(r uint64) bool {
+// ChargeRegularOnly deducts a regular-only cost.
+func (g *GasBudget) ChargeRegularOnly(r uint64) error {
 	if g.RegularGas < r {
-		return false
+		return ErrOutOfGas
 	}
 	g.RegularGas -= r
 	g.UsedRegularGas += r
-	return true
+	return nil
 }
 
 // CanAfford reports whether the running budget can cover the given cost vector

core/vm/gen/gen.go

@@ -0,0 +1,972 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/printer"
+	"go/token"
+	"os"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// inlineOps selects the opcodes whose handler bodies are spliced inline: the
+// hot, fork-stable opcodes with no dynamic gas. Which handler that is comes
+// from the per-fork tables via vm.GenForks (see deriveSpecs), not from a
+// restated name. Most resolve to a top-level opXxx handler. PUSH3-PUSH32 and
+// DUP1-DUP16 resolve to makePush / makeDup closures, so emitInlineOp splices
+// the factory body with the per-opcode size instead. Opcodes not selected here
+// (or in directCallOps) fall through to the default case, which dispatches via
+// the per-fork table.
+var inlineOps = func() map[byte]bool {
+	m := map[byte]bool{
+		0x01: true, 0x02: true, 0x03: true, 0x04: true, 0x05: true, // ADD MUL SUB DIV SDIV
+		0x06: true, 0x07: true, 0x08: true, 0x09: true, 0x0b: true, // MOD SMOD ADDMOD MULMOD SIGNEXTEND
+		0x10: true, 0x11: true, 0x12: true, 0x13: true, 0x14: true, 0x15: true, // LT GT SLT SGT EQ ISZERO
+		0x16: true, 0x17: true, 0x18: true, 0x19: true, 0x1a: true, // AND OR XOR NOT BYTE
+		0x1b: true, 0x1c: true, 0x1d: true, 0x1e: true, // SHL SHR SAR CLZ
+		0x50: true, 0x56: true, 0x57: true, 0x58: true, 0x59: true, 0x5b: true, // POP JUMP JUMPI PC MSIZE JUMPDEST
+		0x5f: true, 0x60: true, 0x61: true, // PUSH0 PUSH1 PUSH2
+	}
+	for code := 0x62; code <= 0x7f; code++ { // PUSH3-PUSH32
+		m[byte(code)] = true
+	}
+	for code := 0x80; code <= 0x8f; code++ { // DUP1-DUP16
+		m[byte(code)] = true
+	}
+	for code := 0x90; code <= 0x9f; code++ { // SWAP1-SWAP16
+		m[byte(code)] = true
+	}
+	return m
+}()
+
+// directCallOps selects the opcodes (dynamic gas, not inlined) whose handler,
+// dynamic-gas, and memory-size functions are the same across every fork
+// (verified by checkDirectCallStable). They are emitted as direct calls to
+// those functions by name, with the names derived from the per-fork tables,
+// instead of the indirect operation.* pointer calls in the default case. An
+// aliased gas var derives as its underlying function, so MLOAD's charge is
+// emitted as pureMemoryGascost rather than through the gasMLoad func var.
+var directCallOps = map[byte]bool{
+	0x20: true, // KECCAK256
+	0x51: true, // MLOAD
+	0x52: true, // MSTORE
+	0x53: true, // MSTORE8
+}
+
+// opSpec holds the per-opcode facts the generator emits from: the constants
+// (gas, stack bounds, intro fork) and the FuncForPC names of the opcode's
+// handler, dynamic-gas and memory-size functions, all derived from the
+// per-fork tables.
+type opSpec struct {
+	defined  bool
+	name     string
+	fork     string
+	constGas uint64
+	minStack int
+	maxStack int
+	execFn   string
+	dynFn    string
+	memFn    string
+}
+
+type generator struct {
+	fset           *token.FileSet
+	opcodeHandlers map[string]*ast.FuncDecl
+	stackHelpers   map[string]*ast.FuncDecl
+	gasHelpers     map[string]*ast.FuncDecl
+	specs          [256]opSpec
+	buf            *bytes.Buffer
+}
+
+// p is the writer of the generated file. Every line of output is appended
+// to g.buf through it.
+func (g *generator) p(format string, args ...any) {
+	format = strings.TrimRight(strings.TrimPrefix(format, "\n"), " \t")
+	fmt.Fprintf(g.buf, format, args...)
+}
+
+// parseHandlers parses instructions.go, eips.go, stack.go, gascosts.go and
+// interpreter.go. It returns the top-level opXxx handlers by name, the
+// //gen:inline *Stack helper methods, and the gas/memory helper functions
+// (ChargeRegularOnly, computeMemorySize, chargeDynamicGas) whose bodies are
+// spliced into the generated dispatch (all by name).
+func parseHandlers(vmDir string) (fset *token.FileSet, opcodeHandlers, stackHelpers, gasHelpers map[string]*ast.FuncDecl) {
+	fset = token.NewFileSet()
+	opcodeHandlers = map[string]*ast.FuncDecl{}
+	stackHelpers = map[string]*ast.FuncDecl{}
+	gasHelpers = map[string]*ast.FuncDecl{}
+	for _, name := range []string{"instructions.go", "eips.go", "stack.go", "gascosts.go", "interpreter.go"} {
+		path := filepath.Join(vmDir, name)
+		f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
+		if err != nil {
+			fatalf("parse %s: %v", path, err)
+		}
+		for _, decl := range f.Decls {
+			fn, ok := decl.(*ast.FuncDecl)
+			if !ok || fn.Body == nil {
+				continue
+			}
+			switch {
+			case fn.Name.Name == "ChargeRegularOnly" || fn.Name.Name == "computeMemorySize" || fn.Name.Name == "chargeDynamicGas" || fn.Name.Name == "chargeVerkleCodeChunkGas": // spliced gas/memory helpers
+				gasHelpers[fn.Name.Name] = fn
+			case fn.Recv == nil: // top-level opXxx handler
+				opcodeHandlers[fn.Name.Name] = fn
+			case methodReceiver(fn) == "Stack" && hasInlineMarker(fn): // (s *Stack) helper tagged //gen:inline
+				stackHelpers[fn.Name.Name] = fn
+			}
+		}
+	}
+	return fset, opcodeHandlers, stackHelpers, gasHelpers
+}
+
+// methodReceiver returns the receiver type name of a pointer-receiver method
+// (e.g. "Stack" for (s *Stack)), or "" if fn is not such a method.
+func methodReceiver(fn *ast.FuncDecl) string {
+	if fn.Recv == nil || len(fn.Recv.List) != 1 {
+		return ""
+	}
+	star, ok := fn.Recv.List[0].Type.(*ast.StarExpr)
+	if !ok {
+		return ""
+	}
+	id, ok := star.X.(*ast.Ident)
+	if !ok {
+		return ""
+	}
+	return id.Name
+}
+
+// hasInlineMarker reports whether fn is tagged //gen:inline, which marks a stack
+// helper for splicing into the generated dispatch.
+func hasInlineMarker(fn *ast.FuncDecl) bool {
+	if fn.Doc == nil {
+		return false
+	}
+	for _, c := range fn.Doc.List {
+		if c.Text == "//gen:inline" {
+			return true
+		}
+	}
+	return false
+}
+
+var opcodeReturnRe = regexp.MustCompile(`^(\s*)return\s+([^,]+),\s*(.+)$`)
+
+// spliceOpcodeBody returns a named handler's body, rewritten so it can be spliced
+// into the dispatch loop (see rewriteOpcodeReturns). The caller emits it with p.
+func (g *generator) spliceOpcodeBody(handler string) string {
+	fn := g.opcodeHandlers[handler]
+	if fn == nil {
+		fatalf("no handler %q to inline", handler)
+	}
+	return g.rewriteOpcodeReturns(g.inlineStackHelpers(fn.Body.List, nil))
+}
+
+// spliceOpcodeFactoryBody splices the body of the executionFunc closure that a make*
+// factory returns, substituting the factory's parameters with the per-opcode
+// constants in args (positional, matching the factory signature). This lets
+// closure-built handlers (makePush, makeDup) be derived from their single
+// definition rather than restated in the generator. The caller emits the
+// result with p.
+func (g *generator) spliceOpcodeFactoryBody(factory string, args ...int) string {
+	fn := g.opcodeHandlers[factory]
+	if fn == nil {
+		fatalf("no factory %q to inline", factory)
+	}
+	lit := factoryClosure(factory, fn)
+	// Bind the factory parameters to the per-opcode constants, then inline.
+	names := paramNames(fn)
+	if len(names) != len(args) {
+		fatalf("factory %q takes %d params, got %d args", factory, len(names), len(args))
+	}
+	params := map[string]int{}
+	for i, nm := range names {
+		params[nm] = args[i]
+	}
+	return g.rewriteOpcodeReturns(g.inlineStackHelpers(lit.Body.List, params))
+}
+
+// factoryClosure returns the executionFunc literal that a make* factory's body
+// is a single `return func(...) {...}` of.
+func factoryClosure(name string, fn *ast.FuncDecl) *ast.FuncLit {
+	if len(fn.Body.List) != 1 {
+		fatalf("factory %q body is not a single return", name)
+	}
+	ret, ok := fn.Body.List[0].(*ast.ReturnStmt)
+	if !ok || len(ret.Results) != 1 {
+		fatalf("factory %q does not return a single value", name)
+	}
+	lit, ok := ret.Results[0].(*ast.FuncLit)
+	if !ok {
+		fatalf("factory %q does not return a func literal", name)
+	}
+	return lit
+}
+
+// renderAst converts AST statements back to formatted Go source text, the
+// inverse of parsing. It uses the generator's fileset and emits nothing itself
+// (the caller passes the result to p).
+func (g *generator) renderAst(stmts []ast.Stmt) string {
+	var raw bytes.Buffer
+	cfg := printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
+	for _, stmt := range stmts {
+		if err := cfg.Fprint(&raw, g.fset, stmt); err != nil {
+			fatalf("print stmt: %v", err)
+		}
+		raw.WriteByte('\n')
+	}
+	return raw.String()
+}
+
+// rewriteOpcodeReturns rewrites a printed handler body so it runs inside the
+// dispatch loop: the `*pc` dereference becomes the loop's `pc` local, and each
+// `return r0, r1` becomes loop control flow. Success (r1 == nil) advances pc
+// and continues, an error sets err and breaks. (Stack helpers were already
+// inlined by inlineStackHelpers before the body was printed.)
+func (g *generator) rewriteOpcodeReturns(src string) string {
+	src = strings.ReplaceAll(src, "*pc", "pc")
+
+	var out bytes.Buffer
+	for _, line := range strings.Split(src, "\n") {
+		if m := opcodeReturnRe.FindStringSubmatch(line); m != nil {
+			indent, r0, r1 := m[1], strings.TrimSpace(m[2]), strings.TrimSpace(m[3])
+			if r1 == "nil" {
+				out.WriteString(indent + "pc++\n")
+				out.WriteString(indent + "continue mainLoop\n")
+			} else {
+				out.WriteString(indent + "res, err = " + r0 + ", " + r1 + "\n")
+				out.WriteString(indent + "break mainLoop\n")
+			}
+			continue
+		}
+		out.WriteString(line + "\n")
+	}
+	return out.String()
+}
+
+var gasReturnRe = regexp.MustCompile(`^(\s*)return\s+(\S.*)$`)
+
+// rewriteGasReturns rewrites a spliced charge body so it runs as a gas step in
+// the dispatch loop: a `return <err>` becomes the out-of-gas break, and the
+// trailing `return nil` (success) is dropped so the opcode continues.
+func (g *generator) rewriteGasReturns(src string) string {
+	var out bytes.Buffer
+	for _, line := range strings.Split(src, "\n") {
+		if m := gasReturnRe.FindStringSubmatch(line); m != nil {
+			indent, val := m[1], strings.TrimSpace(m[2])
+			if val == "nil" {
+				continue // success: fall through to the rest of the op
+			}
+			out.WriteString(indent + "res, err = nil, " + val + "\n")
+			out.WriteString(indent + "break mainLoop\n")
+			continue
+		}
+		out.WriteString(line + "\n")
+	}
+	return out.String()
+}
+
+// rewriteStepReturns rewrites a spliced gas-step body's (value, error) returns so
+// it runs inline in the dispatch loop: a non-nil error becomes the out-of-gas
+// break; on success the value is assigned to target (or dropped when target is
+// empty) and the op falls through.
+func (g *generator) rewriteStepReturns(src, target string) string {
+	var out bytes.Buffer
+	for _, line := range strings.Split(src, "\n") {
+		if m := opcodeReturnRe.FindStringSubmatch(line); m != nil {
+			indent, val, errVal := m[1], strings.TrimSpace(m[2]), strings.TrimSpace(m[3])
+			if errVal == "nil" {
+				if target != "" {
+					out.WriteString(indent + target + " = " + val + "\n")
+				}
+				continue
+			}
+			out.WriteString(indent + "res, err = nil, " + errVal + "\n")
+			out.WriteString(indent + "break mainLoop\n")
+			continue
+		}
+		out.WriteString(line + "\n")
+	}
+	return out.String()
+}
+
+// stackCall is a matched call to a tagged helper.
+type stackCall struct {
+	helper string      // helper method name
+	lhs    []ast.Expr  // assignment targets, nil for a void call like dup
+	tok    token.Token // the assignment token, := or =
+	args   []ast.Expr  // call arguments (only dup has one)
+}
+
+// matchStackHelper matches a statement that is a single must-expand helper call,
+// in one of the two normalized forms: an assignment `lhs... := scope.Stack.H(args)`
+// or a bare `scope.Stack.H(args)`.
+func (g *generator) matchStackHelper(stmt ast.Stmt) (stackCall, bool) {
+	switch s := stmt.(type) {
+	case *ast.AssignStmt:
+		if len(s.Rhs) == 1 {
+			if h, args, ok := g.stackHelperCall(s.Rhs[0]); ok {
+				return stackCall{helper: h, lhs: s.Lhs, tok: s.Tok, args: args}, true
+			}
+		}
+	case *ast.ExprStmt:
+		if h, args, ok := g.stackHelperCall(s.X); ok {
+			return stackCall{helper: h, args: args}, true
+		}
+	}
+	return stackCall{}, false
+}
+
+// stackHelperCall unwraps scope.Stack.H(args) where H is a must-expand helper.
+func (g *generator) stackHelperCall(e ast.Expr) (helper string, args []ast.Expr, ok bool) {
+	call, isCall := e.(*ast.CallExpr)
+	if !isCall {
+		return "", nil, false
+	}
+	sel, isSel := call.Fun.(*ast.SelectorExpr) // <recv>.H
+	if !isSel || g.stackHelpers[sel.Sel.Name] == nil || !isStackExpr(sel.X) {
+		return "", nil, false
+	}
+	return sel.Sel.Name, call.Args, true
+}
+
+// isStackExpr reports whether e is the stack receiver: the `stack` local or
+// scope.Stack.
+func isStackExpr(e ast.Expr) bool {
+	switch x := e.(type) {
+	case *ast.Ident:
+		return x.Name == "stack"
+	case *ast.SelectorExpr:
+		return x.Sel.Name == "Stack"
+	}
+	return false
+}
+
+// inlineStackHelpers renders a handler body to source, inlining every must-expand
+// helper call and printing other statements unchanged. params maps the factory
+// parameters (makePush/makeDup) to their per-opcode constants.
+func (g *generator) inlineStackHelpers(stmts []ast.Stmt, params map[string]int) string {
+	var out strings.Builder
+	// Walk the handler body one statement at a time. A statement that is a
+	// tagged stack-helper call gets the helper's body spliced in: the generated
+	// dispatch is past Go's big-function inline budget, so the call would not be
+	// inlined otherwise. Every other statement is printed as written.
+	for _, stmt := range stmts {
+		if call, ok := g.matchStackHelper(stmt); ok {
+			// e.g. `x, y := scope.Stack.pop1Peek1()` becomes the body of pop1Peek1.
+			out.WriteString(g.inlineStackHelper(call, params))
+		} else {
+			// A plain statement: print it verbatim, then fill in any makePush or
+			// makeDup factory params with this opcode's constants.
+			out.WriteString(substParams(g.renderAst([]ast.Stmt{stmt}), params))
+		}
+	}
+	return out.String()
+}
+
+// substParams replaces each factory parameter with its constant. It runs only
+// on printed non-helper statements and on helper arguments, never on a helper
+// expansion, so it cannot touch a field like stack.size. The parameter names do
+// not textually overlap, so map order does not affect the result.
+func substParams(src string, params map[string]int) string {
+	for name, val := range params {
+		src = regexp.MustCompile(`\b`+name+`\b`).ReplaceAllString(src, fmt.Sprint(val))
+	}
+	return src
+}
+
+// inlineStackHelper expands one helper call to its stack.go body. The single
+// rule: the helper is straight-line statements then an optional final return
+// whose result count matches the call's targets. Anything else is not in
+// inlinable form and is a hard error (the shape post-condition). The receiver
+// maps to the loop's `stack` local and each parameter to its call argument.
+func (g *generator) inlineStackHelper(call stackCall, params map[string]int) string {
+	fn := g.stackHelpers[call.helper]
+	if fn == nil {
+		fatalf("no stack helper %q to inline", call.helper)
+	}
+	// Peel an optional trailing return off the body.
+	body := fn.Body.List
+	var ret *ast.ReturnStmt
+	if n := len(body); n > 0 {
+		if r, isRet := body[n-1].(*ast.ReturnStmt); isRet {
+			ret, body = r, body[:n-1]
+		}
+	}
+	results := 0
+	if ret != nil {
+		results = len(ret.Results)
+	}
+	if len(call.lhs) != results {
+		fatalf("stack helper %q returns %d values, call assigns %d", call.helper, results, len(call.lhs))
+	}
+	// Map the receiver to the loop local and each parameter to its argument.
+	names := paramNames(fn)
+	if len(names) != len(call.args) {
+		fatalf("stack helper %q takes %d params, call passes %d", call.helper, len(names), len(call.args))
+	}
+	subst := map[string]string{recvName(fn): "stack"}
+	for i, name := range names {
+		subst[name] = substParams(renderInlineExpr(call.args[i], nil), params)
+	}
+	// The leading bookkeeping statements, then bind each return expression to
+	// its assignment target.
+	var out strings.Builder
+	for _, stmt := range body {
+		out.WriteString(renderInlineStmt(stmt, subst) + "\n")
+	}
+	for i, lhs := range call.lhs {
+		out.WriteString(renderInlineExpr(lhs, nil) + " " + call.tok.String() + " " + renderInlineExpr(ret.Results[i], subst) + "\n")
+	}
+	return out.String()
+}
+
+// recvName returns a method's receiver name (e.g. "s").
+func recvName(fn *ast.FuncDecl) string {
+	if names := fn.Recv.List[0].Names; len(names) > 0 {
+		return names[0].Name
+	}
+	return ""
+}
+
+// paramNames returns a function's parameter names, in order.
+func paramNames(fn *ast.FuncDecl) []string {
+	var names []string
+	for _, f := range fn.Type.Params.List {
+		for _, nm := range f.Names {
+			names = append(names, nm.Name)
+		}
+	}
+	return names
+}
+
+// renderInlineStmt prints one helper-body statement with subst applied. Only the
+// statement shapes the helpers use are handled; any other is not inlinable.
+func renderInlineStmt(stmt ast.Stmt, subst map[string]string) string {
+	switch s := stmt.(type) {
+	case *ast.IncDecStmt: // s.inner.top++
+		return renderInlineExpr(s.X, subst) + s.Tok.String()
+	case *ast.AssignStmt: // s.size -= 2, data[x] = data[y], or the swap tuple a, b = b, a
+		if len(s.Lhs) == len(s.Rhs) && len(s.Lhs) >= 1 {
+			lhs := make([]string, len(s.Lhs))
+			rhs := make([]string, len(s.Rhs))
+			for i := range s.Lhs {
+				lhs[i] = renderInlineExpr(s.Lhs[i], subst)
+				rhs[i] = renderInlineExpr(s.Rhs[i], subst)
+			}
+			return strings.Join(lhs, ", ") + " " + s.Tok.String() + " " + strings.Join(rhs, ", ")
+		}
+	}
+	fatalf("inline: unsupported statement %T in stack helper", stmt)
+	return ""
+}
+
+// renderInlineExpr prints one helper-body expression, substituting any
+// identifier found in subst. Only the shapes the helpers use are handled.
+func renderInlineExpr(expr ast.Expr, subst map[string]string) string {
+	switch e := expr.(type) {
+	case *ast.Ident:
+		if r, ok := subst[e.Name]; ok {
+			return r
+		}
+		return e.Name
+	case *ast.BasicLit:
+		return e.Value
+	case *ast.SelectorExpr: // x.field
+		return renderInlineExpr(e.X, subst) + "." + e.Sel.Name
+	case *ast.IndexExpr: // x[i]
+		return renderInlineExpr(e.X, subst) + "[" + renderInlineExpr(e.Index, subst) + "]"
+	case *ast.BinaryExpr: // x op y
+		return renderInlineExpr(e.X, subst) + " " + e.Op.String() + " " + renderInlineExpr(e.Y, subst)
+	case *ast.UnaryExpr: // &x
+		return e.Op.String() + renderInlineExpr(e.X, subst)
+	}
+	fatalf("inline: unsupported expression %T in stack helper", expr)
+	return ""
+}
+
+// deriveSpecs records each opcode's constant values (name, intro fork, static
+// gas, stack bounds) and its handler, dynamic-gas and memory-size function
+// names from the first fork that defines it, then checks that the opcodes
+// chosen for inlining and direct-calling are safe to emit from those specs by
+// verifying they are fork-stable (see checkStable and checkDirectCallStable).
+func (g *generator) deriveSpecs(forks []vm.GenFork) {
+	for code := range 256 {
+		for _, fork := range forks {
+			o := fork.Ops[code]
+			if !o.Defined {
+				continue
+			}
+			g.specs[code] = opSpec{
+				defined:  true,
+				name:     o.Name,
+				fork:     fork.RuleField,
+				constGas: o.ConstantGas,
+				minStack: o.MinStack,
+				maxStack: o.MaxStack,
+				execFn:   o.ExecuteFn,
+				dynFn:    o.DynamicGasFn,
+				memFn:    o.MemorySizeFn,
+			}
+			break // first fork that defines it wins (its intro fork)
+		}
+	}
+
+	// Every inlined opcode must be defined and keep the same handler and static
+	// gas / stack bounds across all forks where it appears. Bail loudly otherwise.
+	for code := range inlineOps {
+		g.checkStable(code, forks)
+	}
+
+	// directCallOps opcodes emit their static gas and stack bounds as constants the
+	// same way, so they must be fork-stable too. Dynamic gas is allowed (it is
+	// charged through the named gas function, not a constant).
+	for code := range directCallOps {
+		g.checkDirectCallStable(code, forks)
+	}
+}
+
+// checkStable verifies an opcode selected for inlining is safe to inline: it must
+// be defined, its handler and its static gas and stack bounds must be the same
+// across every fork it appears in (the body and constants are emitted from the
+// first defining fork's spec), and it must have no dynamic gas, since an inlined
+// op charges only its constant static gas. It bails loudly otherwise.
+func (g *generator) checkStable(code byte, forks []vm.GenFork) {
+	spec := g.specs[code]
+	if !spec.defined {
+		fatalf("opcode %#x selected for inlining but never defined", code)
+	}
+	for _, fork := range forks {
+		o := fork.Ops[code]
+		if !o.Defined {
+			continue
+		}
+		if o.ExecuteFn != spec.execFn || o.ConstantGas != spec.constGas || o.MinStack != spec.minStack || o.MaxStack != spec.maxStack || o.DynamicGasFn != "" {
+			fatalf("opcode %#x (%s) is not fork-stable (fork %s): cannot inline", code, spec.name, fork.Name)
+		}
+	}
+}
+
+// checkDirectCallStable verifies a directCallOps opcode is safe to direct-call. Its static
+// gas and stack bounds must be the same across every fork it appears in (they are
+// emitted as constants), and its handler, gas and memory functions must be the same
+// across those forks too (they are called by the first defining fork's names, so a
+// fork that swapped one would otherwise be missed). Unlike checkStable it allows
+// dynamic gas, which directCallOps ops carry by definition.
+func (g *generator) checkDirectCallStable(code byte, forks []vm.GenFork) {
+	spec := g.specs[code]
+	if !spec.defined {
+		fatalf("opcode %#x (directCallOps) is never defined", code)
+	}
+	for _, fork := range forks {
+		o := fork.Ops[code]
+		if !o.Defined {
+			continue
+		}
+		if o.ConstantGas != spec.constGas || o.MinStack != spec.minStack || o.MaxStack != spec.maxStack {
+			fatalf("opcode %#x (%s) is in directCallOps but not fork-stable (fork %s): static gas or stack bounds vary, cannot emit as constants", code, spec.name, fork.Name)
+		}
+		if o.ExecuteFn != spec.execFn || o.DynamicGasFn != spec.dynFn || o.MemorySizeFn != spec.memFn {
+			fatalf("opcode %#x (%s) is in directCallOps but its functions vary by fork (fork %s): got %s/%s/%s, want %s/%s/%s, cannot direct-call",
+				code, spec.name, fork.Name, o.ExecuteFn, o.DynamicGasFn, o.MemorySizeFn, spec.execFn, spec.dynFn, spec.memFn)
+		}
+	}
+}
+
+// generateStackChecks returns the underflow/overflow guards, mirroring the legacy
+// loop's order (stack validated before gas). minExpr and maxExpr are the
+// stack-bound expressions (constants on the inlined/direct paths,
+// operation.minStack/maxStack in the table path). under and over select which
+// guards to emit, so those paths can omit a guard whose bound is trivial.
+func (g *generator) generateStackChecks(minExpr, maxExpr any, under, over bool) string {
+	switch {
+	case under && over:
+		return fmt.Sprintf(`if sLen := stack.len(); sLen < %v {
+	return nil, &ErrStackUnderflow{stackLen: sLen, required: %v}
+} else if sLen > %v {
+	return nil, &ErrStackOverflow{stackLen: sLen, limit: %v}
+}
+`, minExpr, minExpr, maxExpr, maxExpr)
+	case under:
+		return fmt.Sprintf(`if sLen := stack.len(); sLen < %v {
+	return nil, &ErrStackUnderflow{stackLen: sLen, required: %v}
+}
+`, minExpr, minExpr)
+	case over:
+		return fmt.Sprintf(`if sLen := stack.len(); sLen > %v {
+	return nil, &ErrStackOverflow{stackLen: sLen, limit: %v}
+}
+`, maxExpr, maxExpr)
+	}
+	return ""
+}
+
+// generateStaticGas returns the static-gas charge, spliced call-free from the
+// ChargeRegularOnly body for amount: a constant on the inlined and
+// direct-call paths, operation.constantGas in the table path. The receiver maps
+// to contract.Gas and the method's single uint64 parameter to amount, substituted
+// textually on word boundaries (which cannot touch fields like RegularGas). Its
+// `return <err>` becomes the loop's out-of-gas exit and its trailing `return nil`
+// is dropped so the opcode falls through to its remaining steps (see
+// rewriteGasReturns).
+func (g *generator) generateStaticGas(amount any) string {
+	fn := g.gasHelpers["ChargeRegularOnly"]
+	if fn == nil {
+		fatalf("no ChargeRegularOnly gas helper to inline")
+	}
+	names := paramNames(fn)
+	if len(names) != 1 {
+		fatalf("ChargeRegularOnly takes %d params, want 1", len(names))
+	}
+	src := g.renderAst(fn.Body.List)
+	src = regexp.MustCompile(`\b`+recvName(fn)+`\b`).ReplaceAllString(src, "contract.Gas")
+	src = regexp.MustCompile(`\b`+names[0]+`\b`).ReplaceAllString(src, fmt.Sprint(amount))
+	return g.rewriteGasReturns(src)
+}
+
+// closureSegRe matches the anonymous trailing segments of a closure's
+// FuncForPC name, "func31" or a nested "2".
+var closureSegRe = regexp.MustCompile(`^(func\d+|\d+)$`)
+
+// factoryName returns the factory a closure-built handler was created by
+// (e.g. "makeDup" for "newFrontierInstructionSet.makeDup.func37"), or "" for
+// a plain top-level handler name.
+func factoryName(fn string) string {
+	segs := strings.Split(fn, ".")
+	n := len(segs)
+	for n > 0 && closureSegRe.MatchString(segs[n-1]) {
+		n--
+	}
+	if n == len(segs) || n == 0 {
+		return ""
+	}
+	return segs[n-1]
+}
+
+// emitInlineOp emits an inlined opcode case: the stack and gas guards followed by
+// the spliced opcode body. A fork-introduced opcode wraps that body in a fork gate
+// so it runs only when the opcode is active for the current fork, otherwise the
+// case mirrors the legacy loop's undefined-opcode handling.
+func (g *generator) emitInlineOp(code byte) {
+	spec := g.specs[code]
+	g.p("case %s:\n", spec.name)
+	if spec.fork != "" {
+		g.p("if rules.%s {\n", spec.fork)
+	}
+
+	// stack bounds check
+	g.p("%s", g.generateStackChecks(spec.minStack, spec.maxStack, spec.minStack > 0, spec.maxStack < int(params.StackLimit)))
+
+	// static gas
+	if spec.constGas != 0 {
+		g.p("%s", g.generateStaticGas(spec.constGas))
+	}
+
+	// PUSH1-PUSH32 swap their execute function under EIP-4762 (verkle) to charge
+	// code-chunk gas on the immediate bytes. Defer to the table handler there.
+	// The constant static gas and stack guard above already match.
+	if code >= 0x60 && code <= 0x7f {
+		g.p(`
+			if isEIP4762 {
+				res, err = table[op].execute(&pc, evm, scope)
+				if err != nil {
+					break mainLoop
+				}
+				pc++
+				continue mainLoop
+			}
+		`)
+	}
+
+	// opcode body
+	switch factory := factoryName(spec.execFn); factory {
+	case "makePush": // PUSH3-PUSH32: splice makePush(size, size)
+		n := int(code) - 0x5f
+		g.p("%s", g.spliceOpcodeFactoryBody("makePush", n, n))
+	case "makeDup": // DUP1-DUP16: splice makeDup(n)
+		g.p("%s", g.spliceOpcodeFactoryBody("makeDup", int(code)-0x7f))
+	case "": // the rest: splice the opXxx handler body
+		g.p("%s", g.spliceOpcodeBody(spec.execFn))
+	default:
+		fatalf("opcode %#x (%s) is built by factory %q, which the generator cannot inline", code, spec.name, factory)
+	}
+
+	// If opcode is inactive for this fork, then close the gate
+	// and fall back to the legacy loop's undefined-opcode handling.
+	if spec.fork != "" {
+		g.p(`
+	    }
+		  res, err = opUndefined(&pc, evm, scope)
+			break mainLoop
+		`)
+	}
+}
+
+// emitDirectCallOp emits an opcode case identical to the default case, except
+// the handler, dynamic-gas, and memory-size functions are called by name
+// rather than through the indirect operation.* table pointers. Valid only for
+// fork-invariant ops (see directCallOps).
+func (g *generator) emitDirectCallOp(code byte) {
+	spec := g.specs[code]
+	g.p("case %s:\n", spec.name)
+
+	// stack bounds check
+	g.p("%s", g.generateStackChecks(spec.minStack, spec.maxStack, spec.minStack > 0, spec.maxStack < int(params.StackLimit)))
+
+	// static gas
+	if spec.constGas != 0 {
+		g.p("%s", g.generateStaticGas(spec.constGas))
+	}
+
+	// dynamic gas
+	g.p("\nvar memorySize uint64\n")
+
+	// Splice computeMemorySize's body, rewriting its operation.memorySize lookup to
+	// the opcode's memory-size function and its returns for the dispatch loop.
+	memSizeFn := g.gasHelpers["computeMemorySize"]
+	if memSizeFn == nil {
+		fatalf("no computeMemorySize gas helper to inline")
+	}
+	memSizeSrc := g.renderAst(memSizeFn.Body.List)
+	memSizeSrc = strings.ReplaceAll(memSizeSrc, "operation.memorySize", spec.memFn)
+	g.p("%s", g.rewriteStepReturns(memSizeSrc, "memorySize"))
+
+	// Splice chargeDynamicGas's body the same way, rewriting operation.dynamicGas to
+	// the opcode's gas function.
+	dynGasFn := g.gasHelpers["chargeDynamicGas"]
+	if dynGasFn == nil {
+		fatalf("no chargeDynamicGas gas helper to inline")
+	}
+	dynGasSrc := g.renderAst(dynGasFn.Body.List)
+	dynGasSrc = strings.ReplaceAll(dynGasSrc, "operation.dynamicGas", spec.dynFn)
+	g.p("%s", g.rewriteStepReturns(dynGasSrc, ""))
+
+	// resize memory
+	g.p(`
+		if memorySize > 0 {
+			mem.Resize(memorySize)
+		}
+	`)
+
+	// call the opcode handler
+	g.p(`
+		res, err = %s(&pc, evm, scope)
+		if err != nil {
+			break mainLoop
+		}
+	`, spec.execFn)
+
+	// advance to the next opcode
+	g.p(`
+		pc++
+		continue mainLoop
+	`)
+}
+
+// emitDefault emits the switch's default case: every opcode not inlined or
+// direct-called (the fork-varying ops such as CALL, CREATE, SSTORE, SLOAD, LOG
+// and the COPY family) is dispatched through the active per-fork table, exactly
+// as the legacy loop did, so their volatile gas and opcode logic stays shared
+// rather than restated here.
+func (g *generator) emitDefault() {
+	g.p(`
+		default:
+			operation := table[op]
+	`)
+	// stack bounds check
+	g.p("%s", g.generateStackChecks("operation.minStack", "operation.maxStack", true, true))
+
+	// static gas
+	g.p("%s", g.generateStaticGas("operation.constantGas"))
+
+	// dynamic gas
+	g.p(`
+			var memorySize uint64
+			if memorySize, _, err = contract.meterDynamicGas(operation, evm, stack, mem); err != nil {
+				return nil, err
+			}
+	`)
+
+	// resize memory
+	g.p(`
+			if memorySize > 0 {
+				mem.Resize(memorySize)
+			}
+	`)
+
+	// call the opcode handler
+	g.p(`
+			res, err = operation.execute(&pc, evm, scope)
+			if err != nil {
+				break mainLoop
+			}
+	`)
+
+	// advance to the next opcode
+	g.p(`
+			pc++
+			continue mainLoop
+	`)
+}
+
+// createFile emits the whole generated file into g.buf: the header, package and
+// imports, then the execUntraced function (its locals and dispatch loop, the
+// verkle code-chunk gas, and a switch with one case per opcode built by the
+// emit* helpers). main formats the buffer and writes it to interpreter_gen.go.
+//
+// The switch has three tiers:
+//
+//   - the hot, fork-stable opcodes (arithmetic / comparison / bitwise / PUSH /
+//     DUP / SWAP / POP / JUMP / JUMPI / PC / MSIZE / JUMPDEST) are inlined by
+//     splicing the existing opXxx handler bodies from instructions.go and
+//     eips.go, with their static gas and stack bounds emitted as constants
+//     derived from the per-fork instruction tables via vm.GenForks.
+//
+//   - the fork-invariant ops (KECCAK256 / MLOAD / MSTORE / MSTORE8, see
+//     directCallOps) are called directly by name, skipping the table's function
+//     pointers, which Go cannot inline through.
+//
+//   - everything fork-varying (CALL / CREATE / SSTORE / SLOAD / LOG / the COPY
+//     family and so on) is dispatched through the active per-fork JumpTable in
+//     the default case, exactly as the legacy loop did, so volatile gas and
+//     opcode logic stays shared rather than restated.
+func (g *generator) createFile() {
+	// file header, package clause, and imports
+	g.p(`
+		// Code generated by core/vm/gen; DO NOT EDIT.
+
+		package vm
+
+		import (
+			"fmt"
+
+			"github.com/ethereum/go-ethereum/common/math"
+			"github.com/ethereum/go-ethereum/core/tracing"
+		)
+
+	`)
+
+	// execUntraced: doc comment, loop-local declarations, and the dispatch loop
+	g.p(`
+		// execUntraced is the generated, tracing-free interpreter fast path. Hot,
+		// fork-stable opcodes are inlined with their static gas and stack bounds emitted
+		// as constants. Fork-invariant ops (KECCAK256/MLOAD/MSTORE/MSTORE8) call their
+		// handler and gas functions directly by name. Everything fork-varying is
+		// dispatched through the active per-fork table in the default case. EVM.Run
+		// selects this path when no tracer is configured.
+		func (evm *EVM) execUntraced(scope *ScopeContext) (ret []byte, err error) {
+			var (
+				contract  = scope.Contract
+				mem       = scope.Memory
+				stack     = scope.Stack
+				table     = evm.table
+				rules     = evm.chainRules
+				isEIP4762 = rules.IsEIP4762
+				pc        = uint64(0)
+				res       []byte
+			)
+			_ = mem
+			_ = rules
+			_ = isEIP4762
+			_ = table
+		mainLoop:
+			for {
+	`)
+
+	// verkle code-chunk gas, spliced from chargeVerkleCodeChunkGas
+	ccgFn := g.gasHelpers["chargeVerkleCodeChunkGas"]
+	if ccgFn == nil {
+		fatalf("no chargeVerkleCodeChunkGas gas helper to inline")
+	}
+	g.p("%s", g.rewriteGasReturns(g.renderAst(ccgFn.Body.List)))
+
+	// fetch the opcode and open the dispatch switch
+	g.p(`
+				op := contract.GetOp(pc)
+				switch op {
+	`)
+
+	// one case per inlined or direct-call opcode, in opcode order
+	for code := range 256 {
+		b := byte(code)
+		if inlineOps[b] {
+			g.emitInlineOp(b)
+		} else if directCallOps[b] {
+			g.emitDirectCallOp(b)
+		}
+	}
+
+	// the default case: fork-varying ops via the per-fork table
+	g.emitDefault()
+
+	// close the switch and loop, clear the stop token, and return
+	g.p(`
+				}
+			}
+			if err == errStopToken {
+				err = nil
+			}
+			return res, err
+		}
+	`)
+}
+
+func fatalf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, "gen: "+format+"\n", args...)
+	os.Exit(1)
+}
+
+// vmDir returns the core/vm directory, the parent of this generator package. It
+// is resolved from this source file's own path so it does not depend on the
+// directory the generator or the test happens to run from.
+func vmDir() string {
+	_, self, _, ok := runtime.Caller(0)
+	if !ok {
+		fatalf("cannot resolve generator source path")
+	}
+	return filepath.Dir(filepath.Dir(self)) // .../core/vm/gen -> .../core/vm
+}
+
+// generate parses the opcode, gas and fork definitions under core/vm and returns
+// the formatted contents of interpreter_gen.go. It is the shared core of the
+// generator: main writes the result to disk, and the up-to-date test in
+// gen_test.go compares it against the committed file.
+func generate() ([]byte, error) {
+	fset, opcodeHandlers, stackHelpers, gasHelpers := parseHandlers(vmDir())
+	g := &generator{fset: fset, opcodeHandlers: opcodeHandlers, stackHelpers: stackHelpers, gasHelpers: gasHelpers, buf: new(bytes.Buffer)}
+	g.deriveSpecs(vm.GenForks())
+	g.createFile()
+
+	formatted, err := format.Source(g.buf.Bytes())
+	if err != nil {
+		dbg := filepath.Join(vmDir(), "interpreter_gen.go.broken")
+		os.WriteFile(dbg, g.buf.Bytes(), 0644)
+		return nil, fmt.Errorf("gofmt failed (%v); wrote unformatted output to %s", err, dbg)
+	}
+	return formatted, nil
+}

core/vm/gen/gen_test.go

@@ -0,0 +1,42 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestGeneratedDispatchUpToDate asserts that the committed interpreter_gen.go
+// matches what the generator produces from the current opcode, gas and fork
+// definitions. It is the CI guard against hand-edits to the generated file and
+// against the generator drifting from the committed output.
+func TestGeneratedDispatchUpToDate(t *testing.T) {
+	got, err := generate()
+	if err != nil {
+		t.Fatalf("running generator: %v", err)
+	}
+	want, err := os.ReadFile(filepath.Join(vmDir(), "interpreter_gen.go"))
+	if err != nil {
+		t.Fatalf("reading committed interpreter_gen.go: %v", err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatal("interpreter_gen.go is out of date; run `go generate ./core/vm/...` and commit the result")
+	}
+}

core/vm/gen/main.go

@@ -0,0 +1,41 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Command gen generates core/vm/interpreter_gen.go, the EVM interpreter's untraced
+// fast-path dispatch, a switch over the opcode byte. The generated file is
+// committed and a CI test asserts it matches `go generate` output. Do not
+// hand-edit interpreter_gen.go.
+//
+// Usage: go generate ./core/vm/...
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+func main() {
+	formatted, err := generate()
+	if err != nil {
+		fatalf("%v", err)
+	}
+	out := filepath.Join(vmDir(), "interpreter_gen.go")
+	if err := os.WriteFile(out, formatted, 0644); err != nil {
+		fatalf("write %s: %v", out, err)
+	}
+	fmt.Fprintf(os.Stderr, "gen: wrote %s (%d bytes)\n", out, len(formatted))
+}

core/vm/genspec.go

@@ -0,0 +1,162 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package vm
+
+//go:generate go run ./gen
+
+import (
+	"reflect"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// This file exposes the interpreter's opcode metadata to the code generator in
+// core/vm/gen. It is not used at runtime. It exists so the generator can derive
+// the per-opcode spec (static gas, stack bounds, the fork an opcode first
+// appears in, and the FuncForPC names of its handler/gas/memory functions) from
+// the existing per-fork instruction sets, rather than restating that metadata.
+//
+// The function names supply the generator's opcode-to-handler mapping and its
+// fork-invariance checks. The fork-varying gas/execute functions themselves are
+// still reached through the active per-fork JumpTable at runtime (see
+// interpreter_gen.go), not emitted by name: several are closures (gasCall, the
+// memoryCopierGas family, makeGasLog) that have no callable name.
+
+// GenOp is the generator-facing scalar metadata for one opcode slot in one fork.
+type GenOp struct {
+	Name         string // opcode mnemonic, e.g. "ADD" (valid only if Defined)
+	Defined      bool   // false if the slot is undefined/invalid in this fork
+	ConstantGas  uint64
+	MinStack     int
+	MaxStack     int
+	ExecuteFn    string // FuncForPC name of op.execute
+	DynamicGasFn string // FuncForPC name of op.dynamicGas, "" if nil
+	MemorySizeFn string // FuncForPC name of op.memorySize, "" if nil
+}
+
+// GenFork bundles a fork's name, the params.Rules bool field that activates it
+// (empty for Frontier, which is always active), and its per-opcode metadata.
+type GenFork struct {
+	Name      string
+	RuleField string
+	Ops       [256]GenOp
+}
+
+// codegenSkippedForks are forks in geth's fork schedule that the generator does
+// not give a lane. Verkle/UBT is the only one: over its Shanghai base it adds no
+// opcodes, it only swaps gas and execute functions on existing ones (enable4762),
+// which the generated switch picks up from the active table at runtime. Emitting
+// a lane for it would trip the generator's fork-stability check (an inlined op's
+// execute function is not allowed to vary by fork).
+var codegenSkippedForks = map[string]bool{"IsUBT": true}
+
+// genFnName returns the FuncForPC name of a jump-table function value with the
+// package path stripped (e.g. "gasKeccak256"), or "" if nil. An aliased var
+// resolves to the underlying function (gasMLoad reports "pureMemoryGascost").
+// A closure keeps its enclosing chain (DUP7's handler reports
+// "newFrontierInstructionSet.makeDup.func37"), so the generator can tell which
+// factory built it and unrelated closures cannot collide on a bare "funcN".
+func genFnName(fn any) string {
+	v := reflect.ValueOf(fn)
+	if !v.IsValid() || v.IsNil() {
+		return ""
+	}
+	full := runtime.FuncForPC(v.Pointer()).Name()
+	if i := strings.LastIndex(full, "/"); i >= 0 {
+		full = full[i+1:] // strip the package path, leaving "vm.<name>"
+	}
+	if i := strings.Index(full, "."); i >= 0 {
+		full = full[i+1:] // strip the package name
+	}
+	return full
+}
+
+// GenForks returns per-fork opcode metadata for the interpreter code generator
+// (core/vm/gen), one entry per fork that changes the opcode table, oldest to
+// newest. It derives the progression from params.Rules and LookupInstructionSet
+// (in params.Rules declaration order, which is chronological) so new forks are
+// picked up without restating a list here.
+func GenForks() []GenFork {
+	// Frontier is always active and carries no rule gate.
+	frontier, _ := LookupInstructionSet(params.Rules{})
+	out := []GenFork{genFork("Frontier", "", &frontier)}
+
+	rt := reflect.TypeOf(params.Rules{})
+	for i := range rt.NumField() {
+		field := rt.Field(i)
+		if field.Type.Kind() != reflect.Bool || codegenSkippedForks[field.Name] {
+			continue
+		}
+		// Activate only this field so the fork resolves to the table it gates.
+		var rules params.Rules
+		reflect.ValueOf(&rules).Elem().Field(i).SetBool(true)
+		set, _ := LookupInstructionSet(rules)
+		if sameOps(&set, &frontier) {
+			continue // this rule does not change the opcode table
+		}
+		out = append(out, genFork(strings.TrimPrefix(field.Name, "Is"), field.Name, &set))
+	}
+	return out
+}
+
+// genFork extracts the generator-facing per-opcode metadata from one fork's
+// instruction set.
+func genFork(name, rule string, set *JumpTable) GenFork {
+	gf := GenFork{Name: name, RuleField: rule}
+	for code := range 256 {
+		op := set[code]
+		if op == nil || op.undefined {
+			continue
+		}
+		gf.Ops[code] = GenOp{
+			Name:         OpCode(code).String(),
+			Defined:      true,
+			ConstantGas:  op.constantGas,
+			MinStack:     op.minStack,
+			MaxStack:     op.maxStack,
+			ExecuteFn:    genFnName(op.execute),
+			DynamicGasFn: genFnName(op.dynamicGas),
+			MemorySizeFn: genFnName(op.memorySize),
+		}
+	}
+	return gf
+}
+
+// sameOps reports whether two instruction sets carry identical per-opcode static
+// metadata: which slots are defined, and their static gas and stack bounds. It
+// is used to tell whether a fork actually changes the opcode table. Handler,
+// dynamic-gas and memory-size functions are ignored (they cannot be compared for
+// equality and are reached through the table at runtime).
+func sameOps(a, b *JumpTable) bool {
+	for code := range 256 {
+		oa, ob := a[code], b[code]
+		undefA := oa == nil || oa.undefined
+		undefB := ob == nil || ob.undefined
+		if undefA != undefB {
+			return false
+		}
+		if undefA {
+			continue
+		}
+		if oa.constantGas != ob.constantGas || oa.minStack != ob.minStack || oa.maxStack != ob.maxStack {
+			return false
+		}
+	}
+	return true
+}

core/vm/instructions.go

@@ -536,12 +536,14 @@ func opJumpdest(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 }
 
 func opPc(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.get().SetUint64(*pc)
+	elem := scope.Stack.get()
+	elem.SetUint64(*pc)
 	return nil, nil
 }
 
 func opMsize(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	scope.Stack.get().SetUint64(uint64(scope.Memory.Len()))
+	elem := scope.Stack.get()
+	elem.SetUint64(uint64(scope.Memory.Len()))
 	return nil, nil
 }
 
@@ -1131,10 +1133,8 @@ func makeLog(size int) executionFunc {
 
 // opPush1 is a specialized version of pushN
 func opPush1(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	var (
-		codeLen = uint64(len(scope.Contract.Code))
-		elem    = scope.Stack.get()
-	)
+	codeLen := uint64(len(scope.Contract.Code))
+	elem := scope.Stack.get()
 	*pc += 1
 	if *pc < codeLen {
 		elem.SetUint64(uint64(scope.Contract.Code[*pc]))
@@ -1146,10 +1146,8 @@ func opPush1(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
 
 // opPush2 is a specialized version of pushN
 func opPush2(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) {
-	var (
-		codeLen = uint64(len(scope.Contract.Code))
-		elem    = scope.Stack.get()
-	)
+	codeLen := uint64(len(scope.Contract.Code))
+	elem := scope.Stack.get()
 	if *pc+2 < codeLen {
 		elem.SetBytes2(scope.Contract.Code[*pc+1 : *pc+3])
 	} else if *pc+1 < codeLen {

core/vm/interpreter.go

@@ -92,6 +92,9 @@ func (ctx *ScopeContext) ContractCode() []byte {
 // It's important to note that any errors returned by the interpreter should be
 // considered a revert-and-consume-all-gas operation except for
 // ErrExecutionReverted which means revert-and-keep-gas-left.
+//
+// Run sets up the per-call scope and picks the interpreter loop. Without a
+// tracer it runs the generated execUntraced, otherwise execTraced.
 func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) {
 	// Increment the call depth which is restricted to 1024
 	evm.depth++
@@ -112,17 +115,48 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte
 	if len(contract.Code) == 0 {
 		return nil, nil
 	}
+	contract.Input = input
 
+	mem := NewMemory()         // bound memory
+	stack := evm.arena.stack() // local stack
+	scope := &ScopeContext{
+		Memory:   mem,
+		Stack:    stack,
+		Contract: contract,
+	}
+
+	// The exit hooks in execTraced still need the stack and memory, so this
+	// teardown must run after them. Defers in execTraced fire before this one,
+	// so the ordering holds.
+	defer func() {
+		stack.release()
+		mem.Free()
+	}()
+
+	// The generated fast path resolves its fork gates at generate time, so it
+	// cannot see table changes made by ExtraEips. Those configs run the table
+	// loop, as do tracing and the differential test via forceTableLoop.
+	if evm.forceTableLoop || len(evm.Config.ExtraEips) > 0 || evm.Config.Tracer != nil {
+		return evm.execTraced(scope)
+	}
+	return evm.execUntraced(scope)
+}
+
+// execTraced is the old table-walking interpreter loop. It dispatches every
+// opcode through the per-fork operation table and emits the tracing hooks
+// exactly as before. It stays hand-written so trace output does not change.
+//
+// The differential test also runs this loop without a tracer and compares it
+// against the generated execUntraced, which keeps the two from drifting
+// apart.
+func (evm *EVM) execTraced(scope *ScopeContext) (ret []byte, err error) {
 	var (
 		op          OpCode     // current opcode
 		jumpTable   *JumpTable = evm.table
-		mem                    = NewMemory()       // bound memory
-		stack                  = evm.arena.stack() // local stack
-		callContext            = &ScopeContext{
-			Memory:   mem,
-			Stack:    stack,
-			Contract: contract,
-		}
+		mem                    = scope.Memory
+		stack                  = scope.Stack
+		contract               = scope.Contract
+		callContext            = scope
 		// For optimisation reason we're using uint64 as the program counter.
 		// It's theoretically possible to go above 2^64. The YP defines the PC
 		// to be uint256. Practically much less so feasible.
@@ -136,15 +170,6 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte
 		debug     = evm.Config.Tracer != nil
 		isEIP4762 = evm.chainRules.IsEIP4762
 	)
-	// Don't move this deferred function, it's placed before the OnOpcode-deferred method,
-	// so that it gets executed _after_: the OnOpcode needs the stacks before
-	// they are returned to the pools
-	defer func() {
-		stack.release()
-		mem.Free()
-	}()
-	contract.Input = input
-
 	if debug {
 		defer func() { // this deferred method handles exit-with-error
 			if err == nil {
@@ -168,16 +193,8 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte
 			// Capture pre-execution values for tracing.
 			logged, pcCopy, gasCopy = false, pc, contract.Gas.RegularGas
 		}
-
-		if isEIP4762 && !contract.IsDeployment && !contract.IsSystemCall {
-			// if the PC ends up in a new "chunk" of verkleized code, charge the
-			// associated costs.
-			contractAddr := contract.Address()
-			consumed, wanted := evm.TxContext.AccessEvents.CodeChunksRangeGas(contractAddr, pc, 1, uint64(len(contract.Code)), false, contract.Gas.RegularGas)
-			contract.chargeRegular(consumed, evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk)
-			if consumed < wanted {
-				return nil, ErrOutOfGas
-			}
+		if err = contract.chargeVerkleCodeChunkGas(evm, pc, isEIP4762); err != nil {
+			return nil, err
 		}
 
 		// Get the operation from the jump table and validate the stack to ensure there are
@@ -192,43 +209,19 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte
 			return nil, &ErrStackOverflow{stackLen: sLen, limit: operation.maxStack}
 		}
 		// for tracing: this gas consumption event is emitted below in the debug section.
-		if !contract.Gas.ChargeRegularOnly(cost) {
-			return nil, ErrOutOfGas
+		if err := contract.Gas.ChargeRegularOnly(cost); err != nil {
+			return nil, err
 		}
 
-		// All ops with a dynamic memory usage also has a dynamic gas cost.
+		// All ops with a dynamic memory usage also has a dynamic gas cost. Size the
+		// memory and charge dynamic gas here.
 		var memorySize uint64
-		if operation.dynamicGas != nil {
-			// calculate the new memory size and expand the memory to fit
-			// the operation
-			// Memory check needs to be done prior to evaluating the dynamic gas portion,
-			// to detect calculation overflows
-			if operation.memorySize != nil {
-				memSize, overflow := operation.memorySize(stack)
-				if overflow {
-					return nil, ErrGasUintOverflow
-				}
-				// memory is expanded in words of 32 bytes. Gas
-				// is also calculated in words.
-				if memorySize, overflow = math.SafeMul(toWordSize(memSize), 32); overflow {
-					return nil, ErrGasUintOverflow
-				}
-			}
-			// Consume the gas and return an error if not enough gas is available.
-			// cost is explicitly set so that the capture state defer method can get the proper cost
-			var dynamicCost GasCosts
-			dynamicCost, err = operation.dynamicGas(evm, contract, stack, mem, memorySize)
-			cost += dynamicCost.RegularGas // for tracing
-			if err != nil {
-				return nil, fmt.Errorf("%w: %v", ErrOutOfGas, err)
-			}
-			if dynamicCost.StateGas == 0 {
-				if !contract.Gas.ChargeRegularOnly(dynamicCost.RegularGas) {
-					return nil, ErrOutOfGas
-				}
-			} else if !contract.Gas.charge(dynamicCost) {
-				return nil, ErrOutOfGas
-			}
+		var dynamicCost GasCosts
+		memorySize, dynamicCost, err = contract.meterDynamicGas(operation, evm, stack, mem)
+		// cost is explicitly set so that the capture state defer method can get the proper cost
+		cost += dynamicCost.RegularGas // for tracing
+		if err != nil {
+			return nil, err
 		}
 
 		// Do tracing before potential memory expansion
@@ -263,3 +256,79 @@ func (evm *EVM) Run(contract *Contract, input []byte, readOnly bool) (ret []byte
 
 	return res, err
 }
+
+// computeMemorySize runs an operation's memorySize function and word-aligns the
+// result, guarding overflow. The traced loop and the default case call it, and the
+// direct-call ops splice it, substituting the opcode's memory-size function for
+// operation.memorySize.
+func computeMemorySize(operation *operation, stack *Stack) (uint64, error) {
+	memSize, overflow := operation.memorySize(stack)
+	if overflow {
+		return 0, ErrGasUintOverflow
+	}
+	// memory is expanded in words of 32 bytes. Gas is also calculated in words.
+	size, overflow := math.SafeMul(toWordSize(memSize), 32)
+	if overflow {
+		return 0, ErrGasUintOverflow
+	}
+	return size, nil
+}
+
+// chargeDynamicGas runs an operation's dynamicGas function, treats a computation
+// failure as out of gas, and charges the cost. It returns the cost so the traced
+// loop can report it. The traced loop and the default case call it, and the
+// direct-call ops splice it, substituting the opcode's gas function for
+// operation.dynamicGas.
+func (contract *Contract) chargeDynamicGas(operation *operation, evm *EVM, stack *Stack, mem *Memory, memorySize uint64) (GasCosts, error) {
+	dynamicCost, gerr := operation.dynamicGas(evm, contract, stack, mem, memorySize)
+	if gerr != nil {
+		return dynamicCost, fmt.Errorf("%w: %v", ErrOutOfGas, gerr)
+	}
+	// A regular-only deduction when there is no state gas, otherwise the full
+	// multidimensional charge through the reservoir.
+	if dynamicCost.StateGas == 0 {
+		if cerr := contract.Gas.ChargeRegularOnly(dynamicCost.RegularGas); cerr != nil {
+			return dynamicCost, cerr
+		}
+	} else if !contract.Gas.charge(dynamicCost) {
+		return dynamicCost, ErrOutOfGas
+	}
+	return dynamicCost, nil
+}
+
+// meterDynamicGas sizes the memory an operation needs and charges its dynamic
+// gas, returning the size to expand to and the charged cost (which the traced
+// loop reports). The caller expands the memory afterward. computeMemorySize and
+// chargeDynamicGas do the work, guarded here for the generic table paths. The
+// direct-call ops splice those two by name and skip the guards.
+func (contract *Contract) meterDynamicGas(operation *operation, evm *EVM, stack *Stack, mem *Memory) (memorySize uint64, dynamicCost GasCosts, err error) {
+	if operation.dynamicGas != nil {
+		if operation.memorySize != nil {
+			if memorySize, err = computeMemorySize(operation, stack); err != nil {
+				return memorySize, dynamicCost, err
+			}
+		}
+		if dynamicCost, err = contract.chargeDynamicGas(operation, evm, stack, mem, memorySize); err != nil {
+			return memorySize, dynamicCost, err
+		}
+	}
+	return memorySize, dynamicCost, nil
+}
+
+// chargeVerkleCodeChunkGas charges EIP-4762 (verkle) witness gas for the code chunk the
+// pc sits in. It is a no-op outside verkle and for deployment or system calls.
+// isEIP4762 is passed in so the caller can hoist evm.chainRules.IsEIP4762 out of
+// the loop. The traced loop calls it, the generated loop splices its body.
+func (contract *Contract) chargeVerkleCodeChunkGas(evm *EVM, pc uint64, isEIP4762 bool) error {
+	// if the PC ends up in a new "chunk" of verkleized code, charge the
+	// associated costs.
+	if isEIP4762 && !contract.IsDeployment && !contract.IsSystemCall {
+		contractAddr := contract.Address()
+		consumed, wanted := evm.TxContext.AccessEvents.CodeChunksRangeGas(contractAddr, pc, 1, uint64(len(contract.Code)), false, contract.Gas.RegularGas)
+		contract.chargeRegular(consumed, evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk)
+		if consumed < wanted {
+			return ErrOutOfGas
+		}
+	}
+	return nil
+}

core/vm/interpreter_gen_test.go

@@ -0,0 +1,515 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package vm
+
+// Tests for the generated interpreter dispatch (interpreter_gen.go): that it
+// behaves identically to the table loop, and that the fast path keeps its cheap
+// stack helpers inlined. The check that the committed file matches the generator
+// output lives with the generator, in core/vm/gen.
+
+import (
+	"bytes"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"math/big"
+	"os/exec"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/tracing"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/holiman/uint256"
+)
+
+// Differential test comparing the table loop against the generated dispatch.
+//
+// These tests prove that the generated dispatch (execUntraced) is bit-identical
+// to the table-walking loop (execTraced, run here without a tracer via
+// EVM.forceTableLoop) for the observable surface of an EVM execution: return
+// data, gas left, error/halt, refund counter, emitted logs, and the resulting
+// state root. It runs the same program through both interpreters over freshly
+// built, identical state across several forks, plus a fuzz target over
+// arbitrary bytecode.
+//
+// execTraced is also the production tracing path, and
+// if it drifted from the generated dispatch then traced re-execution would
+// disagree with what consensus executed. Hook emission itself is covered by
+// the tracer test suites instead.
+
+// diffForks is the set of fork configurations the diff test runs every program
+// under. Spanning forks exercises the generated fork gates (e.g. SHL from
+// Constantinople, PUSH0 from Shanghai, CLZ from Osaka) in both the active and
+// the not-yet-activated states.
+var diffForks = func() []struct {
+	name   string
+	cfg    *params.ChainConfig
+	merged bool
+} {
+	// preConstantinople: Byzantium active, Constantinople and later not.
+	preCon := *params.TestChainConfig
+	preCon.ConstantinopleBlock = nil
+	preCon.PetersburgBlock = nil
+	preCon.IstanbulBlock = nil
+	preCon.MuirGlacierBlock = nil
+	preCon.BerlinBlock = nil
+	preCon.LondonBlock = nil
+	preCon.ArrowGlacierBlock = nil
+	preCon.GrayGlacierBlock = nil
+
+	// amsterdam: Merged plus the Amsterdam (EIP-8037) timestamp, so the diff test
+	// exercises the multidimensional gas accounting (regular + state gas). Without
+	// this lane a state-gas charging divergence between the two interpreters would
+	// go unnoticed.
+	ams := *params.MergedTestChainConfig
+	amsTime := uint64(0)
+	ams.AmsterdamTime = &amsTime
+
+	return []struct {
+		name   string
+		cfg    *params.ChainConfig
+		merged bool
+	}{
+		{"Frontier", params.NonActivatedConfig, false},
+		{"Byzantium", &preCon, false},
+		{"London", params.TestChainConfig, false},
+		{"Merged", params.MergedTestChainConfig, true},
+		{"Amsterdam", &ams, true},
+	}
+}()
+
+var (
+	diffContractAddr = common.HexToAddress("0x000000000000000000000000000000000000c0de")
+	diffCalleeAddr   = common.HexToAddress("0x000000000000000000000000000000000000ca11")
+	diffCaller       = common.HexToAddress("0x000000000000000000000000000000000000face")
+)
+
+// diffCalleeCode is deployed at diffCalleeAddr as a CALL/CREATE target: it
+// writes a storage slot, logs, and returns 32 bytes of memory.
+//
+//	PUSH1 0x2a PUSH1 0x07 SSTORE          // sstore(7, 42)
+//	PUSH1 0xbb PUSH1 0x00 MSTORE          // mem[0..32] = 0xbb
+//	PUSH1 0x20 PUSH1 0x00 LOG0            // log0(mem[0:32])
+//	PUSH1 0x20 PUSH1 0x00 RETURN          // return mem[0:32]
+var diffCalleeCode = []byte{
+	byte(PUSH1), 0x2a, byte(PUSH1), 0x07, byte(SSTORE),
+	byte(PUSH1), 0xbb, byte(PUSH1), 0x00, byte(MSTORE),
+	byte(PUSH1), 0x20, byte(PUSH1), 0x00, byte(LOG0),
+	byte(PUSH1), 0x20, byte(PUSH1), 0x00, byte(RETURN),
+}
+
+// asm is a tiny helper to build bytecode from opcodes/immediates.
+func asm(parts ...any) []byte {
+	var b []byte
+	for _, p := range parts {
+		switch v := p.(type) {
+		case OpCode:
+			b = append(b, byte(v))
+		case byte:
+			b = append(b, v)
+		case int:
+			b = append(b, byte(v))
+		case []byte:
+			b = append(b, v...)
+		default:
+			panic("asm: bad part")
+		}
+	}
+	return b
+}
+
+// diffPrograms is a curated set of bytecode snippets covering the inlined hot
+// opcodes, the volatile call-through opcodes, fork-gated opcodes, control flow,
+// and the principal error paths.
+var diffPrograms = []struct {
+	name string
+	code []byte
+	gas  uint64
+}{
+	{"arith", asm(PUSH1, 0x07, PUSH1, 0x03, ADD, PUSH1, 0x02, MUL, PUSH1, 0x04, SUB, PUSH1, 0x03, DIV, PUSH1, 0x05, MOD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"signed-arith", asm(PUSH1, 0x07, PUSH1, 0xfd, SDIV, PUSH1, 0x03, SMOD, PUSH1, 0x02, SIGNEXTEND, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"addmod-mulmod-exp", asm(PUSH1, 0x07, PUSH1, 0x05, PUSH1, 0x03, ADDMOD, PUSH1, 0x09, PUSH1, 0x04, PUSH1, 0x02, MULMOD, PUSH1, 0x03, PUSH1, 0x02, EXP, ADD, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"cmp", asm(PUSH1, 0x07, PUSH1, 0x03, LT, PUSH1, 0x01, GT, PUSH1, 0x01, SLT, PUSH1, 0x01, SGT, PUSH1, 0x01, EQ, ISZERO, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"bitwise", asm(PUSH1, 0xf0, PUSH1, 0x0f, AND, PUSH1, 0xaa, OR, PUSH1, 0x55, XOR, NOT, PUSH1, 0x01, BYTE, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"shifts-clz", asm(PUSH1, 0xff, PUSH1, 0x04, SHL, PUSH1, 0x02, SHR, PUSH1, 0x01, SAR, CLZ, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"dup-swap", asm(PUSH1, 0x01, PUSH1, 0x02, PUSH1, 0x03, DUP3, SWAP2, DUP1, SWAP1, POP, ADD, ADD, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"push0-push32", asm(PUSH0, PUSH3, 0x01, 0x02, 0x03, ADD, PUSH5, 0x01, 0x02, 0x03, 0x04, 0x05, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"keccak", asm(PUSH1, 0x20, PUSH1, 0x00, KECCAK256, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"memory", asm(PUSH1, 0xab, PUSH1, 0x00, MSTORE8, PUSH1, 0xcd, PUSH2, 0x00, 0x40, MSTORE, MSIZE, PUSH1, 0x60, MSTORE, PUSH1, 0x80, PUSH1, 0x00, RETURN), 100000},
+	{"mcopy", asm(PUSH1, 0xff, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, PUSH1, 0x20, MCOPY, PUSH1, 0x40, PUSH1, 0x00, RETURN), 100000},
+	{"storage", asm(PUSH1, 0x63, PUSH1, 0x07, SSTORE, PUSH1, 0x07, SLOAD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"transient", asm(PUSH1, 0x63, PUSH1, 0x07, TSTORE, PUSH1, 0x07, TLOAD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"loop", asm(PUSH1, 0x00, JUMPDEST, PUSH1, 0x01, ADD, DUP1, PUSH1, 0x05, LT, PUSH1, 0x02, JUMPI, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"jump", asm(PUSH1, 0x06, JUMP, INVALID, INVALID, JUMPDEST, PUSH1, 0x2a, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"env", asm(ADDRESS, CALLER, CALLVALUE, ORIGIN, GASPRICE, CODESIZE, GAS, PC, ADD, ADD, ADD, ADD, ADD, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"block", asm(NUMBER, TIMESTAMP, COINBASE, GASLIMIT, CHAINID, SELFBALANCE, BASEFEE, DIFFICULTY, ADD, ADD, ADD, ADD, ADD, ADD, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"calldata", asm(PUSH1, 0x00, CALLDATALOAD, CALLDATASIZE, PUSH1, 0x00, PUSH1, 0x00, CALLDATACOPY, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"codecopy", asm(PUSH1, 0x10, PUSH1, 0x00, PUSH1, 0x00, CODECOPY, PUSH1, 0x10, PUSH1, 0x00, RETURN), 100000},
+	{"log", asm(PUSH1, 0x11, PUSH1, 0x00, MSTORE, PUSH1, 0x22, PUSH1, 0x33, PUSH1, 0x20, PUSH1, 0x00, LOG2, STOP), 100000},
+	// Fuzz-found regression (the stale-res bug): a res-setting DELEGATECALL
+	// followed by a halting inlined op (JUMPI to an invalid destination). The
+	// buggy build returned the DELEGATECALL output instead of nil.
+	{"delegatecall-then-invalid-jumpi", asm(
+		PUSH1, 0x30, PUSH1, 0x30, PUSH1, 0x30, PUSH1, 0x30,
+		PUSH20, diffCalleeAddr.Bytes(),
+		PUSH2, 0x30, 0x30, DELEGATECALL,
+		PC, PC, JUMPI), 100000},
+	{"extaccess", asm(PUSH20, diffCalleeAddr.Bytes(), EXTCODESIZE, PUSH20, diffCalleeAddr.Bytes(), BALANCE, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 100000},
+	{"call", asm(
+		PUSH1, 0x20, PUSH1, 0x00, PUSH1, 0x00, PUSH1, 0x00, PUSH1, 0x00,
+		PUSH20, diffCalleeAddr.Bytes(), PUSH2, 0xff, 0xff, CALL,
+		PUSH1, 0x20, PUSH1, 0x00, RETURN), 200000},
+	{"staticcall", asm(
+		PUSH1, 0x20, PUSH1, 0x00, PUSH1, 0x00, PUSH1, 0x00,
+		PUSH20, diffCalleeAddr.Bytes(), PUSH2, 0xff, 0xff, STATICCALL,
+		PUSH1, 0x20, PUSH1, 0x00, RETURN), 200000},
+	{"delegatecall", asm(
+		PUSH1, 0x20, PUSH1, 0x00, PUSH1, 0x00, PUSH1, 0x00,
+		PUSH20, diffCalleeAddr.Bytes(), PUSH2, 0xff, 0xff, DELEGATECALL,
+		PUSH1, 0x20, PUSH1, 0x00, RETURN), 200000},
+	{"create", asm(
+		// store init code that returns empty, then CREATE
+		PUSH1, 0x00, PUSH1, 0x00, MSTORE,
+		PUSH1, 0x00, PUSH1, 0x00, PUSH1, 0x00, CREATE,
+		PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 200000},
+	{"revert", asm(PUSH1, 0xaa, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, REVERT), 100000},
+	{"selfdestruct", asm(PUSH20, diffCaller.Bytes(), SELFDESTRUCT), 100000},
+	{"stop", asm(PUSH1, 0x01, STOP), 100000},
+	{"invalid-opcode", asm(PUSH1, 0x01, INVALID), 100000},
+	{"undefined-opcode", asm(PUSH1, 0x01, 0x0c), 100000},
+	{"stack-underflow", asm(ADD), 100000},
+	{"oog", asm(PUSH1, 0x07, PUSH1, 0x03, ADD, PUSH1, 0x00, MSTORE, PUSH1, 0x20, PUSH1, 0x00, RETURN), 7},
+	{"invalid-jump", asm(PUSH1, 0x03, JUMP, STOP), 100000},
+}
+
+// diffResult captures the observable outcome of running a program.
+type diffResult struct {
+	ret     []byte
+	gasLeft uint64
+	errStr  string // "" if no error
+	refund  uint64
+	root    common.Hash
+	logs    []*types.Log
+}
+
+func (r diffResult) equal(o diffResult) (string, bool) {
+	if !bytes.Equal(r.ret, o.ret) {
+		return "return data", false
+	}
+	if r.gasLeft != o.gasLeft {
+		return "gas left", false
+	}
+	if r.errStr != o.errStr {
+		return "error", false
+	}
+	if r.refund != o.refund {
+		return "refund", false
+	}
+	if r.root != o.root {
+		return "state root", false
+	}
+	if len(r.logs) != len(o.logs) {
+		return "log count", false
+	}
+	for i := range r.logs {
+		a, b := r.logs[i], o.logs[i]
+		if a.Address != b.Address || !bytes.Equal(a.Data, b.Data) || len(a.Topics) != len(b.Topics) {
+			return "log content", false
+		}
+		for j := range a.Topics {
+			if a.Topics[j] != b.Topics[j] {
+				return "log topic", false
+			}
+		}
+	}
+	return "", true
+}
+
+func newDiffState(t testing.TB) *state.StateDB {
+	t.Helper()
+	statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+	// Main contract: balance + a pre-set storage slot.
+	statedb.CreateAccount(diffContractAddr)
+	statedb.SetBalance(diffContractAddr, uint256.NewInt(1000), tracing.BalanceChangeUnspecified)
+	statedb.SetState(diffContractAddr, common.Hash{31: 0x07}, common.Hash{31: 0x07})
+	// Callee target for CALL/STATICCALL/DELEGATECALL.
+	statedb.CreateAccount(diffCalleeAddr)
+	statedb.SetBalance(diffCalleeAddr, uint256.NewInt(500), tracing.BalanceChangeUnspecified)
+	statedb.SetCode(diffCalleeAddr, diffCalleeCode, tracing.CodeChangeUnspecified)
+	// Caller EOA with a balance.
+	statedb.CreateAccount(diffCaller)
+	statedb.SetBalance(diffCaller, uint256.NewInt(1<<62), tracing.BalanceChangeUnspecified)
+	statedb.Finalise(true)
+	return statedb
+}
+
+func diffBlockCtx(merged bool) BlockContext {
+	ctx := BlockContext{
+		CanTransfer: func(StateDB, common.Address, *uint256.Int) bool { return true },
+		Transfer:    func(StateDB, common.Address, common.Address, *uint256.Int, *params.Rules) {},
+		GetHash:     func(uint64) common.Hash { return common.Hash{0xde, 0xad} },
+		Coinbase:    common.HexToAddress("0xc01ba5e"),
+		BlockNumber: big.NewInt(8),
+		Time:        1234,
+		Difficulty:  big.NewInt(0x20000),
+		GasLimit:    30_000_000,
+		BaseFee:     big.NewInt(7),
+		BlobBaseFee: big.NewInt(3),
+		// Price state gas as mainnet does (see core.NewEVMBlockContext), so the
+		// Amsterdam diff lane exercises the EIP-8037 state-gas charging path.
+		CostPerStateByte: params.CostPerStateByte,
+	}
+	if merged {
+		h := common.HexToHash("0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20")
+		ctx.Random = &h
+	}
+	return ctx
+}
+
+// TestExtraEIPs checks that EIPs enabled via Config.ExtraEips take effect even
+// when they touch opcodes the generated dispatch inlines. PUSH0 (EIP-3855) on
+// a pre-Shanghai config is the canary: the runtime table has it enabled but
+// the generated fork gate does not, so execution must route through the table loop.
+func TestExtraEIPs(t *testing.T) {
+	code := asm(PUSH0, STOP)
+	statedb := newDiffState(t)
+	statedb.SetCode(diffContractAddr, code, tracing.CodeChangeUnspecified)
+	statedb.Finalise(true)
+
+	evm := NewEVM(diffBlockCtx(false), statedb, params.TestChainConfig, Config{ExtraEips: []int{3855}})
+	evm.SetTxContext(TxContext{
+		Origin:   diffCaller,
+		GasPrice: uint256.NewInt(1),
+	})
+	_, _, err := evm.Call(diffCaller, diffContractAddr, nil, NewGasBudget(100000, 0), new(uint256.Int))
+	if err != nil {
+		t.Fatalf("PUSH0 enabled via ExtraEips failed: %v", err)
+	}
+}
+
+// runOne executes code at diffContractAddr with the given interpreter selection
+// and returns the observable result.
+func runOne(t testing.TB, cfg *params.ChainConfig, merged, useTableLoop bool, code, input []byte, gas uint64) diffResult {
+	t.Helper()
+	statedb := newDiffState(t)
+	statedb.SetCode(diffContractAddr, code, tracing.CodeChangeUnspecified)
+	statedb.Finalise(true)
+
+	evm := NewEVM(diffBlockCtx(merged), statedb, cfg, Config{})
+	evm.SetTxContext(TxContext{
+		Origin:     diffCaller,
+		GasPrice:   uint256.NewInt(1),
+		BlobHashes: []common.Hash{{0xb1, 0x0b}},
+	})
+	evm.forceTableLoop = useTableLoop
+
+	ret, leftOver, err := evm.Call(diffCaller, diffContractAddr, input, NewGasBudget(gas, 0), new(uint256.Int))
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+	return diffResult{
+		ret:     ret,
+		gasLeft: leftOver.RegularGas,
+		errStr:  errStr,
+		refund:  statedb.GetRefund(),
+		root:    statedb.IntermediateRoot(true),
+		logs:    statedb.Logs(),
+	}
+}
+
+func TestInterpreterDiff(t *testing.T) {
+	for _, fk := range diffForks {
+		for _, prog := range diffPrograms {
+			t.Run(fk.name+"/"+prog.name, func(t *testing.T) {
+				input := common.FromHex("0xdeadbeef00000000000000000000000000000000000000000000000000000042")
+				table := runOne(t, fk.cfg, fk.merged, true, prog.code, input, prog.gas)
+				gen := runOne(t, fk.cfg, fk.merged, false, prog.code, input, prog.gas)
+				if where, ok := gen.equal(table); !ok {
+					t.Fatalf("divergence in %s:\n  table: ret=%x gas=%d err=%q refund=%d root=%x logs=%d\n  gen:   ret=%x gas=%d err=%q refund=%d root=%x logs=%d",
+						where,
+						table.ret, table.gasLeft, table.errStr, table.refund, table.root, len(table.logs),
+						gen.ret, gen.gasLeft, gen.errStr, gen.refund, gen.root, len(gen.logs))
+				}
+			})
+		}
+	}
+}
+
+// FuzzInterpreterDiff fuzzes arbitrary bytecode + calldata + gas and asserts the
+// generated dispatch matches the table-walking loop on every observable axis.
+func FuzzInterpreterDiff(f *testing.F) {
+	for _, prog := range diffPrograms {
+		f.Add(prog.code, []byte{0x01, 0x02, 0x03, 0x04}, uint64(100000))
+	}
+	// A couple of structurally-interesting seeds.
+	f.Add(asm(PUSH1, 0x00, JUMPDEST, PUSH1, 0x01, ADD, DUP1, PUSH1, 0xff, GT, PUSH1, 0x02, JUMPI, STOP), []byte{}, uint64(50000))
+	f.Add(bytes.Repeat([]byte{byte(PUSH1), 0x01}, 64), []byte{}, uint64(100000))
+
+	f.Fuzz(func(t *testing.T, code, input []byte, gas uint64) {
+		if len(code) > 24576 { // max contract code size, keep cases realistic
+			return
+		}
+		if gas > 5_000_000 {
+			gas = 5_000_000 // bound execution time
+		}
+		for _, fk := range diffForks {
+			table := runOne(t, fk.cfg, fk.merged, true, code, input, gas)
+			gen := runOne(t, fk.cfg, fk.merged, false, code, input, gas)
+			if where, ok := gen.equal(table); !ok {
+				t.Fatalf("divergence in %s (fork %s): code=%x input=%x gas=%d\n  table: ret=%x gas=%d err=%q refund=%d root=%x logs=%d\n  gen:   ret=%x gas=%d err=%q refund=%d root=%x logs=%d",
+					where, fk.name, code, input, gas,
+					table.ret, table.gasLeft, table.errStr, table.refund, table.root, len(table.logs),
+					gen.ret, gen.gasLeft, gen.errStr, gen.refund, gen.root, len(gen.logs))
+			}
+		}
+	})
+}
+
+// markedHelpers parses stack.go and returns the *Stack helpers tagged
+// //gen:inline. That tag is the single source of truth, shared with the
+// generator (core/vm/gen), for which helpers are spliced into the dispatch.
+func markedHelpers(t *testing.T) map[string]bool {
+	t.Helper()
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "stack.go", nil, parser.ParseComments)
+	if err != nil {
+		t.Fatalf("parsing stack.go: %v", err)
+	}
+	marked := map[string]bool{}
+	for _, decl := range f.Decls {
+		fn, ok := decl.(*ast.FuncDecl)
+		if !ok || fn.Doc == nil {
+			continue
+		}
+		for _, c := range fn.Doc.List {
+			if c.Text == "//gen:inline" {
+				marked[fn.Name.Name] = true
+			}
+		}
+	}
+	if len(marked) == 0 {
+		t.Fatal("found no //gen:inline helpers in stack.go")
+	}
+	return marked
+}
+
+// TestGeneratedFastPathHelpersExpanded asserts the generator spliced every
+// //gen:inline helper inline, so none survives as a real call in interpreter_gen.go.
+// Those helpers exceed the compiler's inline budget for a function as large as
+// execUntraced, so a missed splice would silently drop the inlining the fast
+// path exists for. It is the expand-side counterpart to
+// TestGeneratedFastPathHelpersInlined: together they hold the one invariant that
+// the fast path makes no real stack-helper call, the costly ones by splicing,
+// the cheap ones by compiler inlining.
+func TestGeneratedFastPathHelpersExpanded(t *testing.T) {
+	calls := countStackCalls(t, "interpreter_gen.go")
+	for h := range markedHelpers(t) {
+		if n := calls[h]; n != 0 {
+			t.Errorf("(*Stack).%s is //gen:inline but has %d residual call(s) in interpreter_gen.go, expected 0.\n"+
+				"The generator did not splice it. Check it is still in inlinable shape (core/vm/gen).", h, n)
+		}
+	}
+}
+
+// TestGeneratedFastPathHelpersInlined recompiles this package with the
+// compiler's inlining diagnostics on and fails if any *Stack helper call that
+// survives into interpreter_gen.go was not inlined. Every survivor must be a cheap
+// helper (len, pop1, peek, drop) the compiler inlines into execUntraced; the
+// //gen:inline helpers are spliced away and owned by the Expanded test. The
+// cheap ones inline today with margin except pop1, at cost 18 against Go's
+// big-function budget of 20. A toolchain that re-scores inline cost, or an extra
+// branch in one of these bodies, could silently stop the inlining and slow the
+// interpreter, so this turns that into a red build.
+func TestGeneratedFastPathHelpersInlined(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping inlining check (recompiles the package) in -short mode")
+	}
+
+	// go build -gcflags=-m prints every inlining decision. The build cache
+	// replays the diagnostics on a hit, so repeated runs are deterministic. The
+	// flag applies only to this package, cached dependencies stay quiet.
+	out, err := exec.Command("go", "build", "-gcflags=-m", ".").CombinedOutput()
+	if err != nil {
+		t.Fatalf("compiling with inlining diagnostics: %v\n%s", err, out)
+	}
+	diag := string(out)
+	if !strings.Contains(diag, "interpreter_gen.go") {
+		t.Fatalf("captured no interpreter_gen.go diagnostics, the -m build produced nothing to check:\n%s", diag)
+	}
+
+	// Every surviving stack-helper call (i.e. not a //gen:inline target) must be
+	// inlined by the compiler.
+	marked := markedHelpers(t)
+	for h, n := range countStackCalls(t, "interpreter_gen.go") {
+		if marked[h] {
+			continue // spliced away, owned by TestGeneratedFastPathHelpersExpanded
+		}
+		inlinedRe := regexp.MustCompile(`interpreter_gen\.go.*inlining call to \(\*Stack\)\.` + regexp.QuoteMeta(h) + `\b`)
+		inlined := len(inlinedRe.FindAllString(diag, -1))
+		if inlined != n {
+			t.Errorf("(*Stack).%s: %d call site(s) in interpreter_gen.go, %d inlined into execUntraced.\n"+
+				"The compiler stopped inlining it, so the fast path now pays a real call. Shrink the\n"+
+				"body to fit the inline budget, or tag it //gen:inline in stack.go to splice it instead.", h, n, inlined)
+			continue
+		}
+		t.Logf("(*Stack).%s: %d/%d call sites inlined", h, inlined, n)
+	}
+}
+
+// countStackCalls parses a generated source file and counts calls to each
+// *Stack helper method, keyed by method name. It matches the fast path's stack
+// local and scope.Stack receivers. Parsing rather than grepping keeps comments
+// and strings from inflating the count.
+func countStackCalls(t *testing.T, file string) map[string]int {
+	t.Helper()
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, file, nil, 0)
+	if err != nil {
+		t.Fatalf("parsing %s: %v", file, err)
+	}
+	counts := map[string]int{}
+	ast.Inspect(f, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		if sel, ok := call.Fun.(*ast.SelectorExpr); ok && isStackReceiver(sel.X) {
+			counts[sel.Sel.Name]++
+		}
+		return true
+	})
+	return counts
+}
+
+// isStackReceiver reports whether x is the fast path's stack local or scope.Stack.
+func isStackReceiver(x ast.Expr) bool {
+	switch r := x.(type) {
+	case *ast.Ident:
+		return r.Name == "stack"
+	case *ast.SelectorExpr:
+		return r.Sel.Name == "Stack"
+	}
+	return false
+}

core/vm/stack.go

@@ -104,11 +104,12 @@ func (s *Stack) push(d *uint256.Int) {
 
 // get returns a pointer to a newly created element
 // on top of the stack
+//
+//gen:inline
 func (s *Stack) get() *uint256.Int {
-	elem := &s.inner.data[s.inner.top]
 	s.inner.top++
 	s.size++
-	return elem
+	return &s.inner.data[s.inner.top-1]
 }
 
 func (s *Stack) pop() uint256.Int {
@@ -137,6 +138,8 @@ func (s *Stack) pop1() *uint256.Int {
 
 // pop2 removes the top two elements and returns pointers to them. The
 // pointers stay valid only until the next push or sub call.
+//
+//gen:inline
 func (s *Stack) pop2() (top, second *uint256.Int) {
 	s.inner.top -= 2
 	s.size -= 2
@@ -162,6 +165,8 @@ func (s *Stack) pop4() (top, second, third, fourth *uint256.Int) {
 // pop1Peek1 removes the top element and returns pointers to it and to the new
 // top, the usual operand and write target of a binary operation. The first
 // pointer stays valid only until the next push or sub call.
+//
+//gen:inline
 func (s *Stack) pop1Peek1() (top, rest *uint256.Int) {
 	s.inner.top--
 	s.size--
@@ -171,61 +176,95 @@ func (s *Stack) pop1Peek1() (top, rest *uint256.Int) {
 // pop2Peek1 removes the top two elements and returns pointers to them and to
 // the new top, for three operand operations. The first two pointers stay
 // valid only until the next push or sub call.
+//
+//gen:inline
 func (s *Stack) pop2Peek1() (top, second, rest *uint256.Int) {
 	s.inner.top -= 2
 	s.size -= 2
 	return &s.inner.data[s.inner.top+1], &s.inner.data[s.inner.top], &s.inner.data[s.inner.top-1]
 }
 
+//gen:inline
 func (s *Stack) swap1() {
 	s.inner.data[s.bottom+s.size-2], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-2]
 }
+
+//gen:inline
 func (s *Stack) swap2() {
 	s.inner.data[s.bottom+s.size-3], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-3]
 }
+
+//gen:inline
 func (s *Stack) swap3() {
 	s.inner.data[s.bottom+s.size-4], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-4]
 }
+
+//gen:inline
 func (s *Stack) swap4() {
 	s.inner.data[s.bottom+s.size-5], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-5]
 }
+
+//gen:inline
 func (s *Stack) swap5() {
 	s.inner.data[s.bottom+s.size-6], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-6]
 }
+
+//gen:inline
 func (s *Stack) swap6() {
 	s.inner.data[s.bottom+s.size-7], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-7]
 }
+
+//gen:inline
 func (s *Stack) swap7() {
 	s.inner.data[s.bottom+s.size-8], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-8]
 }
+
+//gen:inline
 func (s *Stack) swap8() {
 	s.inner.data[s.bottom+s.size-9], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-9]
 }
+
+//gen:inline
 func (s *Stack) swap9() {
 	s.inner.data[s.bottom+s.size-10], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-10]
 }
+
+//gen:inline
 func (s *Stack) swap10() {
 	s.inner.data[s.bottom+s.size-11], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-11]
 }
+
+//gen:inline
 func (s *Stack) swap11() {
 	s.inner.data[s.bottom+s.size-12], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-12]
 }
+
+//gen:inline
 func (s *Stack) swap12() {
 	s.inner.data[s.bottom+s.size-13], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-13]
 }
+
+//gen:inline
 func (s *Stack) swap13() {
 	s.inner.data[s.bottom+s.size-14], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-14]
 }
+
+//gen:inline
 func (s *Stack) swap14() {
 	s.inner.data[s.bottom+s.size-15], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-15]
 }
+
+//gen:inline
 func (s *Stack) swap15() {
 	s.inner.data[s.bottom+s.size-16], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-16]
 }
+
+//gen:inline
 func (s *Stack) swap16() {
 	s.inner.data[s.bottom+s.size-17], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-17]
 }
 
+//gen:inline
 func (s *Stack) dup(n int) {
 	s.inner.data[s.bottom+s.size] = s.inner.data[s.bottom+s.size-n]
 	s.size++