forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-19 19:30:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/vm, core/vm/gen: generator code cleanup and elimination of handcoded bodies for SWAP

Browse source
This commit is contained in:
jonny rhea 2026-06-26 09:59:14 -05:00
parent 7e2d97ce5f
commit 5d1ff24f54
3 changed files with 169 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

228
core/vm/gen/main.go
View file

@ -23,8 +23,8 @@
// eips.go, with their static gas and stack bounds baked in as constants // eips.go, with their static gas and stack bounds baked in as constants
// derived from the per-fork instruction tables via vm.GenForks. // derived from the per-fork instruction tables via vm.GenForks.
// //
// - calls the fork-invariant cold ops (KECCAK256 / MLOAD / MSTORE / MSTORE8, // - calls the fork-invariant ops (KECCAK256 / MLOAD / MSTORE / MSTORE8,
// see directCold) directly by name, skipping the table's function // see directCall) directly by name, skipping the table's function
// pointers, which Go cannot inline through. // pointers, which Go cannot inline through.
// //
// - dispatches everything fork-varying (CALL / CREATE / SSTORE / SLOAD / LOG / // - dispatches everything fork-varying (CALL / CREATE / SSTORE / SLOAD / LOG /
@ -57,12 +57,15 @@ import (
const stackLimit = 1024 // params.StackLimit const stackLimit = 1024 // params.StackLimit
// inlineHandler maps an opcode byte to the opXxx handler whose body is spliced // inlineHandler maps an opcode byte to the handler whose body is spliced inline
// inline for that opcode. These are the hot, fork-stable opcodes with no dynamic // for that opcode. These are the hot, fork-stable opcodes with no dynamic gas.
// gas. Opcodes not listed here (or in directCold, or PUSH3-32 / DUP1-16, which // The value is usually an opXxx handler, but PUSH3-PUSH32 and DUP1-DUP16 are
// are handled specially) fall through to the default case, which dispatches via // factory-built (one shared makePush / makeDup each), so their value is the
// the per-fork table. // factory name and emitOpBody splices the factory body with the per-opcode size.
var inlineHandler = map[byte]string{ // Opcodes not listed here (or in directCall) fall through to the default case,
// which dispatches via the per-fork table.
var inlineHandler = func() map[byte]string {
m := map[byte]string{
0x01: "opAdd", 0x02: "opMul", 0x03: "opSub", 0x04: "opDiv", 0x05: "opSdiv", 0x01: "opAdd", 0x02: "opMul", 0x03: "opSub", 0x04: "opDiv", 0x05: "opSdiv",
0x06: "opMod", 0x07: "opSmod", 0x08: "opAddmod", 0x09: "opMulmod", 0x0b: "opSignExtend", 0x06: "opMod", 0x07: "opSmod", 0x08: "opAddmod", 0x09: "opMulmod", 0x0b: "opSignExtend",
0x10: "opLt", 0x11: "opGt", 0x12: "opSlt", 0x13: "opSgt", 0x14: "opEq", 0x15: "opIszero", 0x10: "opLt", 0x11: "opGt", 0x12: "opSlt", 0x13: "opSgt", 0x14: "opEq", 0x15: "opIszero",
@ -74,15 +77,23 @@ var inlineHandler = map[byte]string{
0x94: "opSwap5", 0x95: "opSwap6", 0x96: "opSwap7", 0x97: "opSwap8", 0x94: "opSwap5", 0x95: "opSwap6", 0x96: "opSwap7", 0x97: "opSwap8",
0x98: "opSwap9", 0x99: "opSwap10", 0x9a: "opSwap11", 0x9b: "opSwap12", 0x98: "opSwap9", 0x99: "opSwap10", 0x9a: "opSwap11", 0x9b: "opSwap12",
0x9c: "opSwap13", 0x9d: "opSwap14", 0x9e: "opSwap15", 0x9f: "opSwap16", 0x9c: "opSwap13", 0x9d: "opSwap14", 0x9e: "opSwap15", 0x9f: "opSwap16",
} }
for code := 0x62; code <= 0x7f; code++ { // PUSH3-PUSH32
m[byte(code)] = "makePush"
}
for code := 0x80; code <= 0x8f; code++ { // DUP1-DUP16
m[byte(code)] = "makeDup"
}
return m
}()
// directCold lists cold opcodes (dynamic gas, not inlined) whose handler, // directCall lists the opcodes (dynamic gas, not inlined) whose handler,
// dynamic-gas, and memory-size functions are the same across every fork // dynamic-gas, and memory-size functions are the same across every fork
// (verified: untouched by any enableXxx). They are emitted as direct calls to // (verified: untouched by any enableXxx). They are emitted as direct calls to
// those functions by name instead of the indirect operation.* pointer calls // those functions by name instead of the indirect operation.* pointer calls
// in the default case. Go inlines the plain functions, and the var-aliased gas // in the default case. Go inlines the plain functions, and the var-aliased gas
// funcs (gasMLoad and friends alias pureMemoryGascost) at least skip the // funcs (gasMLoad and friends alias pureMemoryGascost) at least skip the
// table load. Measured at ~3.4% on snailtracer (p=0.000). Fork-varying cold ops // table load. Measured at ~3.4% on snailtracer (p=0.000). Fork-varying ops
// (CALL/SSTORE/SLOAD and friends) do not qualify and stay on the per-fork // (CALL/SSTORE/SLOAD and friends) do not qualify and stay on the per-fork
// table in the default case. // table in the default case.
// //
@ -91,15 +102,15 @@ var inlineHandler = map[byte]string{
// Limited to the memory/hash ops that appear in hot loops. Adding more (e.g. // Limited to the memory/hash ops that appear in hot loops. Adding more (e.g.
// CALLDATACOPY/RETURN, typically once per call) grows the generated function // CALLDATACOPY/RETURN, typically once per call) grows the generated function
// and regresses tiny benchmarks through code layout, for negligible gain. // and regresses tiny benchmarks through code layout, for negligible gain.
var directCold = map[byte][3]string{ var directCall = map[byte][3]string{
0x20: {"opKeccak256", "gasKeccak256", "memoryKeccak256"}, // KECCAK256 0x20: {"opKeccak256", "gasKeccak256", "memoryKeccak256"}, // KECCAK256
0x51: {"opMload", "gasMLoad", "memoryMLoad"}, // MLOAD 0x51: {"opMload", "gasMLoad", "memoryMLoad"}, // MLOAD
0x52: {"opMstore", "gasMStore", "memoryMStore"}, // MSTORE 0x52: {"opMstore", "gasMStore", "memoryMStore"}, // MSTORE
0x53: {"opMstore8", "gasMStore8", "memoryMStore8"}, // MSTORE8 0x53: {"opMstore8", "gasMStore8", "memoryMStore8"}, // MSTORE8
} }
// opMeta is the per-opcode metadata derived from the per-fork tables. // opSpec holds the per-opcode constants the generator bakes (gas, stack bounds, intro fork), derived from the per-fork tables.
type opMeta struct { type opSpec struct {
defined bool defined bool
name string // opcode mnemonic, e.g. "ADD" name string // opcode mnemonic, e.g. "ADD"
introF string // params.Rules field activating it, empty for Frontier (always on) introF string // params.Rules field activating it, empty for Frontier (always on)
@ -112,19 +123,12 @@ type generator struct {
fset *token.FileSet fset *token.FileSet
handlers map[string]*ast.FuncDecl // opXxx handlers from instructions.go and eips.go handlers map[string]*ast.FuncDecl // opXxx handlers from instructions.go and eips.go
stackHelpers map[string]*ast.FuncDecl // (s *Stack) helpers from stack.go, spliced inline stackHelpers map[string]*ast.FuncDecl // (s *Stack) helpers from stack.go, spliced inline
meta [256]opMeta specs [256]opSpec
buf *bytes.Buffer buf *bytes.Buffer
} }
// p is the sole writer of the generated file. Every line of output is appended // p is the writer of the generated file. Every line of output is appended
// to g.buf through it. The splice helpers (inlineBody, inlineFactoryBody) only // to g.buf through it.
// build text, which their callers then hand to p.
//
// p formats and appends generated Go, like fmt.Fprintf. A template can start on
// the line after p( and be indented to match its structure. p drops the leading
// newline and the indent before the closing backtick, and gofmt tidies the rest.
// Double any percent meant for the generated code (%%w, %%v) so Fprintf emits it
// literally.
func (g *generator) p(format string, args ...any) { func (g *generator) p(format string, args ...any) {
format = strings.TrimRight(strings.TrimPrefix(format, "\n"), " \t") format = strings.TrimRight(strings.TrimPrefix(format, "\n"), " \t")
fmt.Fprintf(g.buf, format, args...) fmt.Fprintf(g.buf, format, args...)
@ -199,7 +203,7 @@ func (g *generator) inlineBody(handler string) string {
if fn == nil { if fn == nil {
fatalf("no handler %q to inline", handler) fatalf("no handler %q to inline", handler)
} }
return g.rewriteSplicedBody(g.expandStackHelpers(fn.Body.List, nil)) return g.rewriteSplicedBody(g.inlineStackHelpers(fn.Body.List, nil))
} }
// inlineFactoryBody splices the body of the executionFunc closure that a make* // inlineFactoryBody splices the body of the executionFunc closure that a make*
@ -223,7 +227,7 @@ func (g *generator) inlineFactoryBody(factory string, args ...int) string {
for i, nm := range names { for i, nm := range names {
params[nm] = args[i] params[nm] = args[i]
} }
return g.rewriteSplicedBody(g.expandStackHelpers(lit.Body.List, params)) return g.rewriteSplicedBody(g.inlineStackHelpers(lit.Body.List, params))
} }
// factoryClosure returns the executionFunc literal that a make* factory's body // factoryClosure returns the executionFunc literal that a make* factory's body
@ -262,7 +266,7 @@ func (g *generator) renderAst(stmts []ast.Stmt) string {
// dispatch loop: the `*pc` dereference becomes the loop's `pc` local, and each // dispatch loop: the `*pc` dereference becomes the loop's `pc` local, and each
// `return r0, r1` becomes loop control flow. Success (r1 == nil) advances pc // `return r0, r1` becomes loop control flow. Success (r1 == nil) advances pc
// and continues, an error sets err and breaks. (Stack helpers were already // and continues, an error sets err and breaks. (Stack helpers were already
// inlined by expandStackHelpers before the body was printed.) // inlined by inlineStackHelpers before the body was printed.)
func (g *generator) rewriteSplicedBody(src string) string { func (g *generator) rewriteSplicedBody(src string) string {
src = strings.ReplaceAll(src, "*pc", "pc") src = strings.ReplaceAll(src, "*pc", "pc")
@ -293,13 +297,6 @@ func (g *generator) rewriteSplicedBody(src string) string {
return out.String() return out.String()
} }
// The helpers spliced inline are tagged //gen:inline in stack.go and collected
// into g.stackHelpers. They cost more than the compiler will inline into a
// function the size of execUntraced, where a snailtracer profile put the calls
// at over a tenth of the run. The cheap helpers (pop1, len, peek, drop, back)
// inline on their own and stay as calls. Each tagged helper is inlined from its
// stack.go body (expandStackHelpers), so the inlined code follows the one definition.
// stackCall is a matched call to a tagged helper. // stackCall is a matched call to a tagged helper.
type stackCall struct { type stackCall struct {
helper string // helper method name helper string // helper method name
@ -352,17 +349,22 @@ func isStackExpr(e ast.Expr) bool {
return false return false
} }
// expandStackHelpers renders a handler body to source, inlining every must-expand // inlineStackHelpers renders a handler body to source, inlining every must-expand
// helper call and printing other statements unchanged. params maps the factory // helper call and printing other statements unchanged. params maps the factory
// parameters (makePush/makeDup) to their per-opcode constants. Helper calls // parameters (makePush/makeDup) to their per-opcode constants.
// appear only at statement top-level here; a nested one would be left as a real func (g *generator) inlineStackHelpers(stmts []ast.Stmt, params map[string]int) string {
// call and caught by the inlining guard test.
func (g *generator) expandStackHelpers(stmts []ast.Stmt, params map[string]int) string {
var out strings.Builder var out strings.Builder
// Walk the handler body one statement at a time. A statement that is a
// tagged stack-helper call gets the helper's body spliced in: the generated
// dispatch is past Go's big-function inline budget, so the call would not be
// inlined otherwise. Every other statement is printed as written.
for _, stmt := range stmts { for _, stmt := range stmts {
if call, ok := g.matchStackHelper(stmt); ok { if call, ok := g.matchStackHelper(stmt); ok {
// e.g. `x, y := scope.Stack.pop1Peek1()` becomes the body of pop1Peek1.
out.WriteString(g.inlineStackHelper(call, params)) out.WriteString(g.inlineStackHelper(call, params))
} else { } else {
// A plain statement: print it verbatim, then fill in any makePush or
// makeDup factory params with this opcode's constants.
out.WriteString(substParams(g.renderAst([]ast.Stmt{stmt}), params)) out.WriteString(substParams(g.renderAst([]ast.Stmt{stmt}), params))
} }
} }
@ -451,9 +453,15 @@ func renderInlineStmt(stmt ast.Stmt, subst map[string]string) string {
switch s := stmt.(type) { switch s := stmt.(type) {
case *ast.IncDecStmt: // s.inner.top++ case *ast.IncDecStmt: // s.inner.top++
return renderInlineExpr(s.X, subst) + s.Tok.String() return renderInlineExpr(s.X, subst) + s.Tok.String()
case *ast.AssignStmt: // s.size -= 2 or data[x] = data[y] case *ast.AssignStmt: // s.size -= 2, data[x] = data[y], or the swap tuple a, b = b, a
if len(s.Lhs) == 1 && len(s.Rhs) == 1 { if len(s.Lhs) == len(s.Rhs) && len(s.Lhs) >= 1 {
return renderInlineExpr(s.Lhs[0], subst) + " " + s.Tok.String() + " " + renderInlineExpr(s.Rhs[0], subst) lhs := make([]string, len(s.Lhs))
rhs := make([]string, len(s.Rhs))
for i := range s.Lhs {
lhs[i] = renderInlineExpr(s.Lhs[i], subst)
rhs[i] = renderInlineExpr(s.Rhs[i], subst)
}
return strings.Join(lhs, ", ") + " " + s.Tok.String() + " " + strings.Join(rhs, ", ")
} }
} }
fatalf("inline: unsupported statement %T in stack helper", stmt) fatalf("inline: unsupported statement %T in stack helper", stmt)
@ -485,17 +493,17 @@ func renderInlineExpr(expr ast.Expr, subst map[string]string) string {
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Metadata derivation (from the per-fork tables, via vm.GenForks) // Spec derivation (from the per-fork tables, via vm.GenForks)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
func (g *generator) deriveMeta(forks []vm.GenFork) { func (g *generator) deriveSpecs(forks []vm.GenFork) {
for code := 0; code < 256; code++ { for code := 0; code < 256; code++ {
for _, fork := range forks { for _, fork := range forks {
o := fork.Ops[code] o := fork.Ops[code]
if !o.Defined { if !o.Defined {
continue continue
} }
g.meta[code] = opMeta{ g.specs[code] = opSpec{
defined: true, defined: true,
name: o.Name, name: o.Name,
introF: fork.RuleField, introF: fork.RuleField,
@ -512,23 +520,17 @@ func (g *generator) deriveMeta(forks []vm.GenFork) {
for code, handler := range inlineHandler { for code, handler := range inlineHandler {
g.checkStable(code, handler, forks) g.checkStable(code, handler, forks)
} }
for code := 0x62; code <= 0x7f; code++ { // PUSH3-32 // directCall opcodes bake their static gas and stack bounds the same way, so
g.checkStable(byte(code), "makePush", forks)
}
for code := 0x80; code <= 0x8f; code++ { // DUP1-16
g.checkStable(byte(code), "makeDup", forks)
}
// directCold opcodes bake their static gas and stack bounds the same way, so
// they must be fork-stable too. Dynamic gas is allowed (it is charged through // they must be fork-stable too. Dynamic gas is allowed (it is charged through
// the named gas function, not baked). // the named gas function, not baked).
for code := range directCold { for code := range directCall {
g.checkColdStable(code, forks) g.checkDirectCallStable(code, forks)
} }
} }
func (g *generator) checkStable(code byte, what string, forks []vm.GenFork) { func (g *generator) checkStable(code byte, what string, forks []vm.GenFork) {
m := g.meta[code] spec := g.specs[code]
if !m.defined { if !spec.defined {
fatalf("opcode %#x (%s) selected for inlining but never defined", code, what) fatalf("opcode %#x (%s) selected for inlining but never defined", code, what)
} }
for _, fork := range forks { for _, fork := range forks {
@ -536,23 +538,23 @@ func (g *generator) checkStable(code byte, what string, forks []vm.GenFork) {
if !o.Defined { if !o.Defined {
continue continue
} }
if o.ConstantGas != m.constGas || o.MinStack != m.minStack || o.MaxStack != m.maxStack || o.DynamicGasFn != "" { if o.ConstantGas != spec.constGas || o.MinStack != spec.minStack || o.MaxStack != spec.maxStack || o.DynamicGasFn != "" {
fatalf("opcode %#x (%s) is not fork-stable (fork %s): cannot inline", code, what, fork.Name) fatalf("opcode %#x (%s) is not fork-stable (fork %s): cannot inline", code, what, fork.Name)
} }
} }
} }
// checkColdStable verifies a directCold opcode is safe to direct-call. Its static // checkDirectCallStable verifies a directCall opcode is safe to direct-call. Its static
// gas and stack bounds must be the same across every fork it appears in (they are // gas and stack bounds must be the same across every fork it appears in (they are
// baked as constants), and its handler, gas and memory functions must be the same // baked as constants), and its handler, gas and memory functions must be the same
// across those forks too (they are called by name, so a fork that swapped one // across those forks too (they are called by name, so a fork that swapped one
// would otherwise be missed). Unlike checkStable it allows dynamic gas, which // would otherwise be missed). Unlike checkStable it allows dynamic gas, which
// directCold ops carry by definition. It does not check the directCold map's names // directCall ops carry by definition. It does not check the directCall map's names
// against the table, which the differential test covers. // against the table, which the differential test covers.
func (g *generator) checkColdStable(code byte, forks []vm.GenFork) { func (g *generator) checkDirectCallStable(code byte, forks []vm.GenFork) {
m := g.meta[code] spec := g.specs[code]
if !m.defined { if !spec.defined {
fatalf("opcode %#x (directCold) is never defined", code) fatalf("opcode %#x (directCall) is never defined", code)
} }
var exec, dyn, mem string var exec, dyn, mem string
seen := false seen := false
@ -561,8 +563,8 @@ func (g *generator) checkColdStable(code byte, forks []vm.GenFork) {
if !o.Defined { if !o.Defined {
continue continue
} }
if o.ConstantGas != m.constGas || o.MinStack != m.minStack || o.MaxStack != m.maxStack { if o.ConstantGas != spec.constGas || o.MinStack != spec.minStack || o.MaxStack != spec.maxStack {
fatalf("opcode %#x (%s) is in directCold but not fork-stable (fork %s): static gas or stack bounds vary, cannot bake", code, m.name, fork.Name) fatalf("opcode %#x (%s) is in directCall but not fork-stable (fork %s): static gas or stack bounds vary, cannot bake", code, spec.name, fork.Name)
} }
// Handler, gas and memory functions must match across forks too, or // Handler, gas and memory functions must match across forks too, or
// direct-calling them by name would skip a fork that swapped one. Names // direct-calling them by name would skip a fork that swapped one. Names
@ -571,8 +573,8 @@ func (g *generator) checkColdStable(code byte, forks []vm.GenFork) {
if !seen { if !seen {
exec, dyn, mem, seen = o.ExecuteFn, o.DynamicGasFn, o.MemorySizeFn, true exec, dyn, mem, seen = o.ExecuteFn, o.DynamicGasFn, o.MemorySizeFn, true
} else if o.ExecuteFn != exec || o.DynamicGasFn != dyn || o.MemorySizeFn != mem { } else if o.ExecuteFn != exec || o.DynamicGasFn != dyn || o.MemorySizeFn != mem {
fatalf("opcode %#x (%s) is in directCold but its functions vary by fork (fork %s): got %s/%s/%s, want %s/%s/%s, cannot direct-call", fatalf("opcode %#x (%s) is in directCall but its functions vary by fork (fork %s): got %s/%s/%s, want %s/%s/%s, cannot direct-call",
code, m.name, fork.Name, o.ExecuteFn, o.DynamicGasFn, o.MemorySizeFn, exec, dyn, mem) code, spec.name, fork.Name, o.ExecuteFn, o.DynamicGasFn, o.MemorySizeFn, exec, dyn, mem)
} }
} }
} }
@ -583,9 +585,9 @@ func (g *generator) checkColdStable(code byte, forks []vm.GenFork) {
// emitStackChecks emits the underflow/overflow guards for a baked opcode, // emitStackChecks emits the underflow/overflow guards for a baked opcode,
// mirroring the legacy loop's order (stack validated before gas). // mirroring the legacy loop's order (stack validated before gas).
func (g *generator) emitStackChecks(m opMeta) { func (g *generator) emitStackChecks(spec opSpec) {
under := m.minStack > 0 under := spec.minStack > 0
over := m.maxStack < stackLimit over := spec.maxStack < stackLimit
switch { switch {
case under && over: case under && over:
g.p(` g.p(`
@ -594,24 +596,24 @@ func (g *generator) emitStackChecks(m opMeta) {
} else if sLen > %d { } else if sLen > %d {
return nil, &ErrStackOverflow{stackLen: sLen, limit: %d} return nil, &ErrStackOverflow{stackLen: sLen, limit: %d}
} }
`, m.minStack, m.minStack, m.maxStack, m.maxStack) `, spec.minStack, spec.minStack, spec.maxStack, spec.maxStack)
case under: case under:
g.p(` g.p(`
if sLen := stack.len(); sLen < %d { if sLen := stack.len(); sLen < %d {
return nil, &ErrStackUnderflow{stackLen: sLen, required: %d} return nil, &ErrStackUnderflow{stackLen: sLen, required: %d}
} }
`, m.minStack, m.minStack) `, spec.minStack, spec.minStack)
case over: case over:
g.p(` g.p(`
if sLen := stack.len(); sLen > %d { if sLen := stack.len(); sLen > %d {
return nil, &ErrStackOverflow{stackLen: sLen, limit: %d} return nil, &ErrStackOverflow{stackLen: sLen, limit: %d}
} }
`, m.maxStack, m.maxStack) `, spec.maxStack, spec.maxStack)
} }
} }
func (g *generator) emitGasCheck(m opMeta) { func (g *generator) emitGasCheck(spec opSpec) {
if m.constGas == 0 { if spec.constGas == 0 {
return return
} }
g.p(` g.p(`
@ -619,15 +621,15 @@ func (g *generator) emitGasCheck(m opMeta) {
return nil, ErrOutOfGas return nil, ErrOutOfGas
} }
contract.Gas.RegularGas -= %d contract.Gas.RegularGas -= %d
`, m.constGas, m.constGas) `, spec.constGas, spec.constGas)
} }
// emitWork emits the stack/gas guards and the opcode body (the portion that runs // emitOpBody emits the stack/gas guards and the opcode body (the portion that runs
// when the opcode is active for the current fork). // when the opcode is active for the current fork).
func (g *generator) emitWork(code byte) { func (g *generator) emitOpBody(code byte) {
m := g.meta[code] spec := g.specs[code]
g.emitStackChecks(m) g.emitStackChecks(spec)
g.emitGasCheck(m) g.emitGasCheck(spec)
// PUSH1-PUSH32 swap their execute function under EIP-4762 (verkle) to charge // PUSH1-PUSH32 swap their execute function under EIP-4762 (verkle) to charge
// code-chunk gas on the immediate bytes. Defer to the table handler there. // code-chunk gas on the immediate bytes. Defer to the table handler there.
@ -645,35 +647,28 @@ func (g *generator) emitWork(code byte) {
`) `)
} }
switch { switch h := inlineHandler[code]; h {
case code >= 0x62 && code <= 0x7f: // PUSH3-PUSH32: splice makePush(n, n) case "makePush": // PUSH3-PUSH32: splice makePush(size, size)
n := int(code) - 0x5f n := int(code) - 0x5f
g.p("%s", g.inlineFactoryBody("makePush", n, n)) g.p("%s", g.inlineFactoryBody("makePush", n, n))
case code >= 0x80 && code <= 0x8f: // DUP1-DUP16: splice makeDup(n) case "makeDup": // DUP1-DUP16: splice makeDup(n)
g.p("%s", g.inlineFactoryBody("makeDup", int(code)-0x7f)) g.p("%s", g.inlineFactoryBody("makeDup", int(code)-0x7f))
case code >= 0x90 && code <= 0x9f: // SWAP1-SWAP16: the swap body, emitted raw default: // the rest: splice the opXxx handler body
n := int(code) - 0x8f g.p("%s", g.inlineBody(h))
g.p(`
stack.inner.data[stack.bottom+stack.size-%d], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-%d]
pc++
continue mainLoop
`, n+1, n+1)
default:
g.p("%s", g.inlineBody(inlineHandler[code]))
} }
} }
func (g *generator) emitInlineCase(code byte) { func (g *generator) emitInlineOp(code byte) {
m := g.meta[code] spec := g.specs[code]
g.p("case %s:\n", m.name) g.p("case %s:\n", spec.name)
if m.introF == "" { if spec.introF == "" {
g.emitWork(code) g.emitOpBody(code)
return return
} }
// Fork-gated: run the inlined body only when the opcode is active for the // Fork-gated: run the inlined body only when the opcode is active for the
// current fork. Otherwise mirror the legacy loop's undefined-opcode handling. // current fork. Otherwise mirror the legacy loop's undefined-opcode handling.
g.p("if rules.%s {\n", m.introF) g.p("if rules.%s {\n", spec.introF)
g.emitWork(code) g.emitOpBody(code)
g.p("}\n") g.p("}\n")
g.p(` g.p(`
res, err = opUndefined(&pc, evm, scope) res, err = opUndefined(&pc, evm, scope)
@ -681,16 +676,16 @@ func (g *generator) emitInlineCase(code byte) {
`) `)
} }
// emitDirectCold emits a cold opcode case identical to the default case, except // emitDirectCallOp emits an opcode case identical to the default case, except
// the handler, dynamic-gas, and memory-size functions are called by name // the handler, dynamic-gas, and memory-size functions are called by name
// rather than through the indirect operation.* table pointers. Valid only for // rather than through the indirect operation.* table pointers. Valid only for
// fork-invariant ops (see directCold). // fork-invariant ops (see directCall).
func (g *generator) emitDirectCold(code byte) { func (g *generator) emitDirectCallOp(code byte) {
m := g.meta[code] spec := g.specs[code]
fns := directCold[code] fns := directCall[code]
g.p("case %s:\n", m.name) g.p("case %s:\n", spec.name)
g.emitStackChecks(m) g.emitStackChecks(spec)
g.emitGasCheck(m) g.emitGasCheck(spec)
// The three %s are the memory-size, dynamic-gas and handler names, in the // The three %s are the memory-size, dynamic-gas and handler names, in the
// order fns[2], fns[1], fns[0]. The doubled %%w and %%v are not generator // order fns[2], fns[1], fns[0]. The doubled %%w and %%v are not generator
// verbs: Fprintf collapses each %% to a single %, so the generated code ends // verbs: Fprintf collapses each %% to a single %, so the generated code ends
@ -799,7 +794,7 @@ func (g *generator) emitFile() {
g.p(` g.p(`
// execUntraced is the generated, tracing-free interpreter fast path. Hot, // execUntraced is the generated, tracing-free interpreter fast path. Hot,
// fork-stable opcodes are inlined with their static gas and stack bounds baked // fork-stable opcodes are inlined with their static gas and stack bounds baked
// in. Fork-invariant cold ops (KECCAK256/MLOAD/MSTORE/MSTORE8) call their // in. Fork-invariant ops (KECCAK256/MLOAD/MSTORE/MSTORE8) call their
// handler and gas functions directly by name. Everything fork-varying is // handler and gas functions directly by name. Everything fork-varying is
// dispatched through the active per-fork table in the default case. EVM.Run // dispatched through the active per-fork table in the default case. EVM.Run
// selects this path when no tracer is configured. // selects this path when no tracer is configured.
@ -831,16 +826,13 @@ func (g *generator) emitFile() {
op := contract.GetOp(pc) op := contract.GetOp(pc)
switch op { switch op {
`) `)
// Inlined hot cases, in opcode order for readability. // Inlined cases, in opcode order for readability.
for code := 0; code < 256; code++ { for code := 0; code < 256; code++ {
b := byte(code) b := byte(code)
_, named := inlineHandler[b] if _, named := inlineHandler[b]; named {
isPushFixed := code >= 0x62 && code <= 0x7f g.emitInlineOp(b)
isDup := code >= 0x80 && code <= 0x8f } else if _, dc := directCall[b]; dc {
if named || isPushFixed || isDup { g.emitDirectCallOp(b)
g.emitInlineCase(b)
} else if _, dc := directCold[b]; dc {
g.emitDirectCold(b)
} }
} }
g.emitDefault() g.emitDefault()
@ -867,7 +859,7 @@ func main() {
fset, handlers, stackHelpers := parseHandlers(vmDir) fset, handlers, stackHelpers := parseHandlers(vmDir)
g := &generator{fset: fset, handlers: handlers, stackHelpers: stackHelpers, buf: new(bytes.Buffer)} g := &generator{fset: fset, handlers: handlers, stackHelpers: stackHelpers, buf: new(bytes.Buffer)}
g.deriveMeta(vm.GenForks()) g.deriveSpecs(vm.GenForks())
g.emitFile() g.emitFile()
formatted, err := format.Source(g.buf.Bytes()) formatted, err := format.Source(g.buf.Bytes())

18
core/vm/interp_gen.go
View file

@ -11,7 +11,7 @@ import (
// execUntraced is the generated, tracing-free interpreter fast path. Hot, // execUntraced is the generated, tracing-free interpreter fast path. Hot,
// fork-stable opcodes are inlined with their static gas and stack bounds baked // fork-stable opcodes are inlined with their static gas and stack bounds baked
// in. Fork-invariant cold ops (KECCAK256/MLOAD/MSTORE/MSTORE8) call their // in. Fork-invariant ops (KECCAK256/MLOAD/MSTORE/MSTORE8) call their
// handler and gas functions directly by name. Everything fork-varying is // handler and gas functions directly by name. Everything fork-varying is
// dispatched through the active per-fork table in the default case. EVM.Run // dispatched through the active per-fork table in the default case. EVM.Run
// selects this path when no tracer is configured. // selects this path when no tracer is configured.
@ -2041,6 +2041,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-2], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-2] stack.inner.data[stack.bottom+stack.size-2], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-2]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP2: case SWAP2:
if sLen := stack.len(); sLen < 3 { if sLen := stack.len(); sLen < 3 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 3} return nil, &ErrStackUnderflow{stackLen: sLen, required: 3}
@ -2052,6 +2053,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-3], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-3] stack.inner.data[stack.bottom+stack.size-3], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-3]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP3: case SWAP3:
if sLen := stack.len(); sLen < 4 { if sLen := stack.len(); sLen < 4 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 4} return nil, &ErrStackUnderflow{stackLen: sLen, required: 4}
@ -2063,6 +2065,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-4], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-4] stack.inner.data[stack.bottom+stack.size-4], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-4]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP4: case SWAP4:
if sLen := stack.len(); sLen < 5 { if sLen := stack.len(); sLen < 5 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 5} return nil, &ErrStackUnderflow{stackLen: sLen, required: 5}
@ -2074,6 +2077,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-5], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-5] stack.inner.data[stack.bottom+stack.size-5], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-5]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP5: case SWAP5:
if sLen := stack.len(); sLen < 6 { if sLen := stack.len(); sLen < 6 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 6} return nil, &ErrStackUnderflow{stackLen: sLen, required: 6}
@ -2085,6 +2089,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-6], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-6] stack.inner.data[stack.bottom+stack.size-6], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-6]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP6: case SWAP6:
if sLen := stack.len(); sLen < 7 { if sLen := stack.len(); sLen < 7 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 7} return nil, &ErrStackUnderflow{stackLen: sLen, required: 7}
@ -2096,6 +2101,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-7], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-7] stack.inner.data[stack.bottom+stack.size-7], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-7]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP7: case SWAP7:
if sLen := stack.len(); sLen < 8 { if sLen := stack.len(); sLen < 8 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 8} return nil, &ErrStackUnderflow{stackLen: sLen, required: 8}
@ -2107,6 +2113,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-8], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-8] stack.inner.data[stack.bottom+stack.size-8], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-8]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP8: case SWAP8:
if sLen := stack.len(); sLen < 9 { if sLen := stack.len(); sLen < 9 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 9} return nil, &ErrStackUnderflow{stackLen: sLen, required: 9}
@ -2118,6 +2125,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-9], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-9] stack.inner.data[stack.bottom+stack.size-9], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-9]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP9: case SWAP9:
if sLen := stack.len(); sLen < 10 { if sLen := stack.len(); sLen < 10 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 10} return nil, &ErrStackUnderflow{stackLen: sLen, required: 10}
@ -2129,6 +2137,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-10], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-10] stack.inner.data[stack.bottom+stack.size-10], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-10]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP10: case SWAP10:
if sLen := stack.len(); sLen < 11 { if sLen := stack.len(); sLen < 11 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 11} return nil, &ErrStackUnderflow{stackLen: sLen, required: 11}
@ -2140,6 +2149,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-11], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-11] stack.inner.data[stack.bottom+stack.size-11], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-11]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP11: case SWAP11:
if sLen := stack.len(); sLen < 12 { if sLen := stack.len(); sLen < 12 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 12} return nil, &ErrStackUnderflow{stackLen: sLen, required: 12}
@ -2151,6 +2161,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-12], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-12] stack.inner.data[stack.bottom+stack.size-12], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-12]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP12: case SWAP12:
if sLen := stack.len(); sLen < 13 { if sLen := stack.len(); sLen < 13 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 13} return nil, &ErrStackUnderflow{stackLen: sLen, required: 13}
@ -2162,6 +2173,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-13], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-13] stack.inner.data[stack.bottom+stack.size-13], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-13]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP13: case SWAP13:
if sLen := stack.len(); sLen < 14 { if sLen := stack.len(); sLen < 14 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 14} return nil, &ErrStackUnderflow{stackLen: sLen, required: 14}
@ -2173,6 +2185,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-14], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-14] stack.inner.data[stack.bottom+stack.size-14], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-14]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP14: case SWAP14:
if sLen := stack.len(); sLen < 15 { if sLen := stack.len(); sLen < 15 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 15} return nil, &ErrStackUnderflow{stackLen: sLen, required: 15}
@ -2184,6 +2197,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-15], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-15] stack.inner.data[stack.bottom+stack.size-15], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-15]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP15: case SWAP15:
if sLen := stack.len(); sLen < 16 { if sLen := stack.len(); sLen < 16 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 16} return nil, &ErrStackUnderflow{stackLen: sLen, required: 16}
@ -2195,6 +2209,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-16], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-16] stack.inner.data[stack.bottom+stack.size-16], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-16]
pc++ pc++
continue mainLoop continue mainLoop
case SWAP16: case SWAP16:
if sLen := stack.len(); sLen < 17 { if sLen := stack.len(); sLen < 17 {
return nil, &ErrStackUnderflow{stackLen: sLen, required: 17} return nil, &ErrStackUnderflow{stackLen: sLen, required: 17}
@ -2206,6 +2221,7 @@ mainLoop:
stack.inner.data[stack.bottom+stack.size-17], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-17] stack.inner.data[stack.bottom+stack.size-17], stack.inner.data[stack.bottom+stack.size-1] = stack.inner.data[stack.bottom+stack.size-1], stack.inner.data[stack.bottom+stack.size-17]
pc++ pc++
continue mainLoop continue mainLoop
default: default:
operation := table[op] operation := table[op]
if sLen := stack.len(); sLen < operation.minStack { if sLen := stack.len(); sLen < operation.minStack {

31
core/vm/stack.go
View file

@ -184,51 +184,82 @@ func (s *Stack) pop2Peek1() (top, second, rest *uint256.Int) {
return &s.inner.data[s.inner.top+1], &s.inner.data[s.inner.top], &s.inner.data[s.inner.top-1] return &s.inner.data[s.inner.top+1], &s.inner.data[s.inner.top], &s.inner.data[s.inner.top-1]
} }
//gen:inline
func (s *Stack) swap1() { func (s *Stack) swap1() {
s.inner.data[s.bottom+s.size-2], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-2] s.inner.data[s.bottom+s.size-2], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-2]
} }
//gen:inline
func (s *Stack) swap2() { func (s *Stack) swap2() {
s.inner.data[s.bottom+s.size-3], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-3] s.inner.data[s.bottom+s.size-3], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-3]
} }
//gen:inline
func (s *Stack) swap3() { func (s *Stack) swap3() {
s.inner.data[s.bottom+s.size-4], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-4] s.inner.data[s.bottom+s.size-4], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-4]
} }
//gen:inline
func (s *Stack) swap4() { func (s *Stack) swap4() {
s.inner.data[s.bottom+s.size-5], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-5] s.inner.data[s.bottom+s.size-5], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-5]
} }
//gen:inline
func (s *Stack) swap5() { func (s *Stack) swap5() {
s.inner.data[s.bottom+s.size-6], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-6] s.inner.data[s.bottom+s.size-6], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-6]
} }
//gen:inline
func (s *Stack) swap6() { func (s *Stack) swap6() {
s.inner.data[s.bottom+s.size-7], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-7] s.inner.data[s.bottom+s.size-7], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-7]
} }
//gen:inline
func (s *Stack) swap7() { func (s *Stack) swap7() {
s.inner.data[s.bottom+s.size-8], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-8] s.inner.data[s.bottom+s.size-8], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-8]
} }
//gen:inline
func (s *Stack) swap8() { func (s *Stack) swap8() {
s.inner.data[s.bottom+s.size-9], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-9] s.inner.data[s.bottom+s.size-9], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-9]
} }
//gen:inline
func (s *Stack) swap9() { func (s *Stack) swap9() {
s.inner.data[s.bottom+s.size-10], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-10] s.inner.data[s.bottom+s.size-10], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-10]
} }
//gen:inline
func (s *Stack) swap10() { func (s *Stack) swap10() {
s.inner.data[s.bottom+s.size-11], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-11] s.inner.data[s.bottom+s.size-11], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-11]
} }
//gen:inline
func (s *Stack) swap11() { func (s *Stack) swap11() {
s.inner.data[s.bottom+s.size-12], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-12] s.inner.data[s.bottom+s.size-12], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-12]
} }
//gen:inline
func (s *Stack) swap12() { func (s *Stack) swap12() {
s.inner.data[s.bottom+s.size-13], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-13] s.inner.data[s.bottom+s.size-13], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-13]
} }
//gen:inline
func (s *Stack) swap13() { func (s *Stack) swap13() {
s.inner.data[s.bottom+s.size-14], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-14] s.inner.data[s.bottom+s.size-14], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-14]
} }
//gen:inline
func (s *Stack) swap14() { func (s *Stack) swap14() {
s.inner.data[s.bottom+s.size-15], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-15] s.inner.data[s.bottom+s.size-15], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-15]
} }
//gen:inline
func (s *Stack) swap15() { func (s *Stack) swap15() {
s.inner.data[s.bottom+s.size-16], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-16] s.inner.data[s.bottom+s.size-16], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-16]
} }
//gen:inline
func (s *Stack) swap16() { func (s *Stack) swap16() {
s.inner.data[s.bottom+s.size-17], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-17] s.inner.data[s.bottom+s.size-17], s.inner.data[s.bottom+s.size-1] = s.inner.data[s.bottom+s.size-1], s.inner.data[s.bottom+s.size-17]
} }