diff --git a/core/bench_test.go b/core/bench_test.go index 47b991e5a3..4e3923e4fe 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -18,7 +18,12 @@ package core import ( "crypto/ecdsa" + "encoding/binary" + "encoding/hex" "math/big" + "os" + "path/filepath" + "strings" "testing" "github.com/ethereum/go-ethereum/common" @@ -68,6 +73,12 @@ func BenchmarkInsertChain_ring1000_memdb(b *testing.B) { func BenchmarkInsertChain_ring1000_diskdb(b *testing.B) { benchInsertChain(b, true, genTxRing(1000)) } +func BenchmarkInsertChain_evmWorkload_memdb(b *testing.B) { + benchInsertChainEVM(b, false) +} +func BenchmarkInsertChain_evmWorkload_diskdb(b *testing.B) { + benchInsertChainEVM(b, true) +} var ( // This is the content of the genesis block used by the benchmarks. @@ -208,6 +219,110 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { } } +// The evmWorkload addresses hold the evm-bench contracts (see +// core/vm/runtime/testdata), installed through the genesis alloc. Their +// runtime bytecode sets up all state inside Benchmark(), so no constructor +// runs. +var benchWorkloadAddrs = []common.Address{ + common.HexToAddress("0x00000000000000000000000000000000000e0001"), // ERC20Transfer + common.HexToAddress("0x00000000000000000000000000000000000e0002"), // ERC20Mint + common.HexToAddress("0x00000000000000000000000000000000000e0003"), // ERC20ApprovalTransfer + common.HexToAddress("0x00000000000000000000000000000000000e0004"), // TenThousandHashes +} + +func benchWorkloadCode(tb testing.TB) [][]byte { + files := []string{"erc20transfer.hex", "erc20mint.hex", "erc20approval.hex", "tenthousandhashes.hex"} + codes := make([][]byte, len(files)) + for i, name := range files { + raw, err := os.ReadFile(filepath.Join("vm", "runtime", "testdata", name)) + if err != nil { + tb.Fatal(err) + } + code, err := hex.DecodeString(strings.TrimSpace(string(raw))) + if err != nil { + tb.Fatal(err) + } + codes[i] = code + } + return codes +} + +// benchInsertChainEVM measures InsertChain over blocks dominated by real +// contract execution, as a coarse local surrogate for sync throughput: the +// timed path is full block processing, including sender recovery, EVM +// execution, state updates, receipts, bloom filters and the state root. +// Every block invokes the four evm-bench workloads (ERC-20 transfer, mint +// and approval loops plus a keccak loop) and sends ten value transfers to +// fresh accounts so the state grows as it would during sync. Reports Mgas/s +// alongside the standard timings. +func benchInsertChainEVM(b *testing.B, disk bool) { + var db ethdb.Database + if !disk { + db = rawdb.NewMemoryDatabase() + } else { + pdb, err := pebble.New(b.TempDir(), 128, 128, "", false) + if err != nil { + b.Fatalf("cannot create temporary database: %v", err) + } + db = rawdb.NewDatabase(pdb) + defer db.Close() + } + alloc := types.GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}} + for i, code := range benchWorkloadCode(b) { + alloc[benchWorkloadAddrs[i]] = types.Account{Code: code, Balance: new(big.Int)} + } + gspec := &Genesis{ + Config: params.TestChainConfig, + Alloc: alloc, + GasLimit: 150_000_000, + } + selector := []byte{0x30, 0x62, 0x7b, 0x7c} // Benchmark() + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), b.N, func(i int, gen *BlockGen) { + signer := gen.Signer() + gasPrice := big.NewInt(0) + if gen.header.BaseFee != nil { + gasPrice = gen.header.BaseFee + } + for _, addr := range benchWorkloadAddrs { + to := addr + tx, _ := types.SignNewTx(benchRootKey, signer, &types.LegacyTx{ + Nonce: gen.TxNonce(benchRootAddr), + To: &to, + Gas: 30_000_000, + Data: selector, + GasPrice: gasPrice, + }) + gen.AddTx(tx) + } + for j := 0; j < 10; j++ { + var to common.Address + binary.BigEndian.PutUint64(to[4:12], uint64(i+1)) + binary.BigEndian.PutUint64(to[12:20], uint64(j+1)) + tx, _ := types.SignNewTx(benchRootKey, signer, &types.LegacyTx{ + Nonce: gen.TxNonce(benchRootAddr), + To: &to, + Value: big.NewInt(1), + Gas: params.TxGas, + GasPrice: gasPrice, + }) + gen.AddTx(tx) + } + }) + var totalGas uint64 + for _, block := range chain { + totalGas += block.GasUsed() + } + chainman, _ := NewBlockChain(db, gspec, ethash.NewFaker(), nil) + defer chainman.Stop() + b.ReportAllocs() + b.ResetTimer() + if i, err := chainman.InsertChain(chain); err != nil { + b.Fatalf("insert error (block %d): %v\n", i, err) + } + b.StopTimer() + b.ReportMetric(float64(totalGas)/1e6/b.Elapsed().Seconds(), "Mgas/s") +} + func BenchmarkChainRead_header_10k(b *testing.B) { benchReadChain(b, false, 10000) } diff --git a/core/vm/runtime/testdata/evm-bench/README.md b/core/vm/runtime/testdata/evm-bench/README.md index cc5ec51620..fccea2ce87 100644 --- a/core/vm/runtime/testdata/evm-bench/README.md +++ b/core/vm/runtime/testdata/evm-bench/README.md @@ -17,7 +17,7 @@ workload: | `ERC20Transfer` | ERC-20 transfers in a loop | | `ERC20Mint` | 5000 `_mint`s (SSTORE/keccak/LOG-heavy) | | `ERC20ApprovalTransfer` | 1000 approve+transferFrom cycles | -| `TenThousandHashes` | a keccak loop | +| `TenThousandHashes` | 20000 chained keccak256 calls | ## Running @@ -40,6 +40,16 @@ interpreter change is committed) and runs `benchstat`. the evm-bench Solidity sources (solc 0.8.17 via docker). These contracts set up their state inside `Benchmark()`, so the runtime bytecode is callable directly. +`TenThousandHashes` is compiled from the vendored +[`TenThousandHashes.sol`](TenThousandHashes.sol) in this directory, not the +upstream evm-bench file. Upstream discards the hash result, so solc's optimizer +deletes the keccak256 and the compiled benchmark degenerates into a bare +counter loop performing a single static hash. The vendored copy chains the hash +through a returned accumulator so the optimized bytecode performs all 20000 +hashes. Numbers for this benchmark are therefore not comparable with evm-bench +or gevm published tables, which use degenerate bytecode (gevm's local +replacement is broken differently, an inverted loop bound that hashes once). + `snailtracer.hex` is **vendored, not regenerated by gen.sh**: evm-bench's `SnailTracer` initializes its scene in the constructor, so its runtime bytecode (no constructor) renders an empty scene. The committed file is a runtime-callable @@ -53,4 +63,5 @@ The `.hex` files are committed, so running the benchmarks needs no toolchain. - The contract constructor is not executed, so storage starts empty. The evm-bench contracts set up whatever state they need inside `Benchmark()`, matching the work the evm-bench harness measures. -- Names match evm-bench / gevm for cross-comparison. +- Names match evm-bench / gevm for cross-comparison, except `TenThousandHashes` + (see the regenerating section). diff --git a/core/vm/runtime/testdata/evm-bench/TenThousandHashes.sol b/core/vm/runtime/testdata/evm-bench/TenThousandHashes.sol new file mode 100644 index 0000000000..004215e96f --- /dev/null +++ b/core/vm/runtime/testdata/evm-bench/TenThousandHashes.sol @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-3.0 +pragma solidity ^0.8.17; + +// Vendored from evm-bench (github.com/ziyadedher/evm-bench) with one fix. +// The upstream loop discards the keccak256 result, so solc's optimizer +// removes the pure call entirely and the compiled benchmark degenerates +// into a bare counter loop that performs a single static hash. Chaining +// the hash through an accumulator that the function returns keeps all +// 20000 hashes in the optimized bytecode. The Benchmark() selector is +// unchanged because return types are not part of the signature. +contract TenThousandHashes { + function Benchmark() external pure returns (bytes32 acc) { + for (uint256 i = 0; i < 20000; i++) { + acc = keccak256(abi.encodePacked(acc, i)); + } + } +} diff --git a/core/vm/runtime/testdata/evm-bench/compare.sh b/core/vm/runtime/testdata/evm-bench/compare.sh index 7ae00603a1..624120bf6e 100755 --- a/core/vm/runtime/testdata/evm-bench/compare.sh +++ b/core/vm/runtime/testdata/evm-bench/compare.sh @@ -28,7 +28,7 @@ COUNT="${2:-10}" # pattern, so the loops run as their own invocation anyway). BENCHES=( '^BenchmarkSnailtracer$:20x' - '^BenchmarkTenThousandHashes$:200x' + '^BenchmarkTenThousandHashes$:100x' '^BenchmarkERC20Transfer$:100x' '^BenchmarkERC20Mint$:150x' '^BenchmarkERC20ApprovalTransfer$:120x' diff --git a/core/vm/runtime/testdata/evm-bench/gen.sh b/core/vm/runtime/testdata/evm-bench/gen.sh index 6b69134823..26dbb6b73d 100755 --- a/core/vm/runtime/testdata/evm-bench/gen.sh +++ b/core/vm/runtime/testdata/evm-bench/gen.sh @@ -8,15 +8,24 @@ # contract's pragma: 0.8.17 for the ERC-20/hashing contracts, 0.4.26 for the # legacy SnailTracer. # +# TenThousandHashes is compiled from the vendored TenThousandHashes.sol in +# this directory rather than the upstream file. Upstream discards the hash +# result, so the optimizer deletes the keccak256 and the benchmark degenerates +# into a bare counter loop. The vendored copy chains the hash through a +# returned accumulator, which keeps all 20000 hashes in the bytecode. See the +# comment in that file. +# # Usage: core/vm/runtime/testdata/evm-bench/gen.sh set -euo pipefail TD="$(cd "$(dirname "$0")/.." && pwd)" # .../core/vm/runtime/testdata +HERE="$(cd "$(dirname "$0")" && pwd)" # .../testdata/evm-bench WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT git clone --depth 1 https://github.com/ziyadedher/evm-bench "$WORK/evm-bench" >/dev/null 2>&1 B="$WORK/evm-bench/benchmarks" +cp "$HERE/TenThousandHashes.sol" "$B/ten-thousand-hashes/TenThousandHashes.sol" # ERC-20 (transfer/mint/approval) + ten-thousand-hashes: pragma ^0.8.17. These # set up all their state inside Benchmark(), so the runtime bytecode is callable diff --git a/core/vm/runtime/testdata/tenthousandhashes.hex b/core/vm/runtime/testdata/tenthousandhashes.hex index 2892c1d984..f2903b13d9 100644 --- a/core/vm/runtime/testdata/tenthousandhashes.hex +++ b/core/vm/runtime/testdata/tenthousandhashes.hex @@ -1 +1 @@ -6080604052348015600f57600080fd5b506004361060285760003560e01c806330627b7c14602d575b600080fd5b60336035565b005b60005b614e20811015606a5760408051602081018390520160408051601f198184030190525280606381606d565b9150506038565b50565b600060018201608c57634e487b7160e01b600052601160045260246000fd5b506001019056fea2646970667358221220e499ea6d1ccd5fce28e8cd2507f6fef33f0d005543e04580f290bbd01df0655464736f6c63430008110033 \ No newline at end of file +6080604052348015600f57600080fd5b506004361060285760003560e01c806330627b7c14602d575b600080fd5b60336045565b60405190815260200160405180910390f35b6000805b614e20811015608e57604080516020810184905290810182905260600160405160208183030381529060405280519060200120915080806087906092565b9150506049565b5090565b60006001820160b157634e487b7160e01b600052601160045260246000fd5b506001019056fea26469706673582212207f65e4c71aef311b81ff8b065cdd78f5be33f5dc80d3b2318e118075e33697f164736f6c63430008110033 \ No newline at end of file