mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-05-09 17:46:37 +00:00
## Summary Replace the `BinaryNode` interface with `NodeRef uint32` indices into typed arena pools, eliminating GC-scanned pointers from binary trie nodes. Inspired by [fjl's observation](https://github.com/ethereum/go-ethereum/pull/34034#issuecomment-4075176446): > *"if the binary trie produces such a large graph, it should probably be changed so that the trie node type does not contain pointers. The runtime does not scan objects that do not contain pointers, so it can really help with the performance to build it this way."* ### The problem CPU profiling of the binary trie (EIP-7864) showed **44% of CPU time in garbage collection**. Each `InternalNode` held two `BinaryNode` interface values (2 pointer-words each), and the GC scanned every one. With ~25K `InternalNode`s in memory during block processing, this created enormous GC pressure. ### The solution `NodeRef` is a compact `uint32` (2-bit kind tag + 30-bit pool index). `NodeStore` manages chunked typed pools per node kind: - **InternalNode pool**: ZERO Go pointers (children are `NodeRef`, hash is `[32]byte`) → noscan spans - **HashedNode pool**: ZERO Go pointers → noscan spans - **StemNode pool**: retains `Values [][]byte` (matching existing format) The serialization format is unchanged — flat InternalNode `[type][leftHash][rightHash]` = 65 bytes. ## Benchmark: Apple M4 Pro (`--benchtime=10s --count=3`, on top of #34021) | Metric | Baseline | Arena | Delta | |--------|----------|-------|-------| | Approve (Mgas/s) | 374 | 382 | **+2.1%** | | BalanceOf (Mgas/s) | 885 | 901 | **+1.8%** | | Approve allocs/op | 775K | **607K** | **-21.7%** | | BalanceOf allocs/op | 265K | **228K** | **-14.0%** | ## Benchmark: AMD EPYC 48-core (50GB state, execution-specs ERC-20, on top of #34021 + #34032) | Benchmark | Baseline | Arena | Delta | |-----------|----------|-------|-------| | erc20_approve (write) | 22.4 Mgas/s | **27.0 Mgas/s** | **+20.5%** | | mixed_sload_sstore | 62.9 Mgas/s | **97.3 Mgas/s** | **+54.7%** | | erc20_balanceof (read) | 180.8 Mgas/s | 167.6 Mgas/s | -7.3% (cold cache variance) | The arena benefit scales with heap size — the EPYC (larger heap, more GC pressure) shows much larger gains than the M4 Pro (efficient unified memory). The mixed workload baseline was unstable (62.9 vs 16.3 Mgas/s between runs due to GC-induced throughput collapse); the arena eliminates this entirely (95-97 Mgas/s, stable). ## Dependencies Benchmarked with #34021 (H01 N+1 fix) + #34032 (R14 parallel hashing). No code dependency — applies independently to master. All test suites pass (`trie/bintrie` with `-race`, `core/state`, `triedb/pathdb`, `cmd/geth`). --------- Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com>
251 lines
8.2 KiB
Go
251 lines
8.2 KiB
Go
// Copyright 2026 go-ethereum Authors
|
|
// This file is part of the go-ethereum library.
|
|
//
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package bintrie
|
|
|
|
import (
|
|
"bytes"
|
|
"testing"
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
"github.com/ethereum/go-ethereum/trie"
|
|
)
|
|
|
|
// makeTrie creates a BinaryTrie populated with the given key-value pairs.
|
|
func makeTrie(t *testing.T, entries [][2]common.Hash) *BinaryTrie {
|
|
t.Helper()
|
|
store := newNodeStore()
|
|
tr := &BinaryTrie{
|
|
store: store,
|
|
tracer: trie.NewPrevalueTracer(),
|
|
}
|
|
for _, kv := range entries {
|
|
if err := store.Insert(kv[0][:], kv[1][:], nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return tr
|
|
}
|
|
|
|
// countLeaves iterates the trie and returns the number of leaves visited.
|
|
func countLeaves(t *testing.T, tr *BinaryTrie) int {
|
|
t.Helper()
|
|
it, err := newBinaryNodeIterator(tr, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
leaves := 0
|
|
for it.Next(true) {
|
|
if it.Leaf() {
|
|
leaves++
|
|
}
|
|
}
|
|
if it.Error() != nil {
|
|
t.Fatalf("iterator error: %v", it.Error())
|
|
}
|
|
return leaves
|
|
}
|
|
|
|
// TestIteratorEmptyTrie verifies that iterating over an empty trie returns
|
|
// no nodes and reports no error.
|
|
func TestIteratorEmptyTrie(t *testing.T) {
|
|
tr := &BinaryTrie{
|
|
store: newNodeStore(),
|
|
tracer: trie.NewPrevalueTracer(),
|
|
}
|
|
it, err := newBinaryNodeIterator(tr, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if it.Next(true) {
|
|
t.Fatal("expected no iteration over empty trie")
|
|
}
|
|
if it.Error() != nil {
|
|
t.Fatalf("unexpected error: %v", it.Error())
|
|
}
|
|
}
|
|
|
|
// TestIteratorSingleStem verifies iteration over a trie with a single stem
|
|
// node containing multiple values.
|
|
func TestIteratorSingleStem(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000003"), oneKey},
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000007"), oneKey},
|
|
{common.HexToHash("00000000000000000000000000000000000000000000000000000000000000FF"), oneKey},
|
|
})
|
|
if leaves := countLeaves(t, tr); leaves != 3 {
|
|
t.Fatalf("expected 3 leaves, got %d", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorTwoStems verifies iteration over a trie with two stems
|
|
// separated by internal nodes, ensuring all leaves from both stems are visited.
|
|
func TestIteratorTwoStems(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002"), oneKey},
|
|
{common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
{common.HexToHash("8000000000000000000000000000000000000000000000000000000000000002"), oneKey},
|
|
})
|
|
if leaves := countLeaves(t, tr); leaves != 4 {
|
|
t.Fatalf("expected 4 leaves, got %d", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorLeafKeyAndBlob verifies that the iterator returns correct
|
|
// leaf keys and values.
|
|
func TestIteratorLeafKeyAndBlob(t *testing.T) {
|
|
key := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000005")
|
|
val := common.HexToHash("00000000000000000000000000000000000000000000000000000000deadbeef")
|
|
tr := makeTrie(t, [][2]common.Hash{{key, val}})
|
|
|
|
it, err := newBinaryNodeIterator(tr, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
found := false
|
|
for it.Next(true) {
|
|
if it.Leaf() {
|
|
found = true
|
|
if !bytes.Equal(it.LeafKey(), key[:]) {
|
|
t.Fatalf("leaf key mismatch: got %x, want %x", it.LeafKey(), key)
|
|
}
|
|
if !bytes.Equal(it.LeafBlob(), val[:]) {
|
|
t.Fatalf("leaf blob mismatch: got %x, want %x", it.LeafBlob(), val)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("expected to find a leaf")
|
|
}
|
|
}
|
|
|
|
// TestIteratorEmptyNodeBacktrack is a regression test for the Empty node
|
|
// backtracking bug. Before the fix, encountering an Empty child during
|
|
// iteration would terminate the walk prematurely instead of backtracking
|
|
// to the parent and continuing with the next sibling.
|
|
func TestIteratorEmptyNodeBacktrack(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
{common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
})
|
|
|
|
if tr.store.root.Kind() != kindInternal {
|
|
t.Fatalf("expected InternalNode root, got kind %d", tr.store.root.Kind())
|
|
}
|
|
if leaves := countLeaves(t, tr); leaves != 2 {
|
|
t.Fatalf("expected 2 leaves, got %d (Empty backtrack bug?)", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorHashedNodeNilData is a regression test for the nil-data guard.
|
|
// When nodeResolver encounters a zero-hash HashedNode, it returns (nil, nil).
|
|
// The iterator should treat this as Empty and continue rather than panicking.
|
|
func TestIteratorHashedNodeNilData(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
{common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
})
|
|
|
|
root := tr.store.root
|
|
if root.Kind() != kindInternal {
|
|
t.Fatalf("expected InternalNode root, got kind %d", root.Kind())
|
|
}
|
|
rootNode := tr.store.getInternal(root.Index())
|
|
|
|
// Replace right child with a zero-hash HashedNode. nodeResolver
|
|
// short-circuits on common.Hash{} and returns (nil, nil), which
|
|
// triggers the nil-data guard in the iterator.
|
|
rootNode.right = tr.store.newHashedRef(common.Hash{})
|
|
|
|
// Should not panic; the zero-hash right child should be treated as Empty.
|
|
// Since the hashed node can't be resolved (nil data -> empty deserialization),
|
|
// only the left leaf should be counted.
|
|
it, err := newBinaryNodeIterator(tr, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
leaves := 0
|
|
for it.Next(true) {
|
|
if it.Leaf() {
|
|
leaves++
|
|
}
|
|
}
|
|
if leaves != 1 {
|
|
t.Fatalf("expected 1 leaf (zero-hash right node skipped), got %d", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorManyStems verifies iteration correctness with many stems,
|
|
// producing a deep tree structure.
|
|
func TestIteratorManyStems(t *testing.T) {
|
|
entries := make([][2]common.Hash, 16)
|
|
for i := range entries {
|
|
var key common.Hash
|
|
key[0] = byte(i << 4)
|
|
key[31] = 1
|
|
entries[i] = [2]common.Hash{key, oneKey}
|
|
}
|
|
tr := makeTrie(t, entries)
|
|
if leaves := countLeaves(t, tr); leaves != 16 {
|
|
t.Fatalf("expected 16 leaves, got %d", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorDeepTree verifies iteration over a trie with stems that share
|
|
// a long common prefix, producing many intermediate InternalNodes.
|
|
func TestIteratorDeepTree(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0C0"), oneKey},
|
|
{common.HexToHash("0000000000E00000000000000000000000000000000000000000000000000000"), twoKey},
|
|
})
|
|
if leaves := countLeaves(t, tr); leaves != 2 {
|
|
t.Fatalf("expected 2 leaves in deep tree, got %d", leaves)
|
|
}
|
|
}
|
|
|
|
// TestIteratorNodeCount verifies the total number of Next(true) calls
|
|
// for a known tree structure.
|
|
func TestIteratorNodeCount(t *testing.T) {
|
|
tr := makeTrie(t, [][2]common.Hash{
|
|
{common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
{common.HexToHash("8000000000000000000000000000000000000000000000000000000000000001"), oneKey},
|
|
})
|
|
|
|
it, err := newBinaryNodeIterator(tr, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
total := 0
|
|
leaves := 0
|
|
for it.Next(true) {
|
|
total++
|
|
if it.Leaf() {
|
|
leaves++
|
|
}
|
|
}
|
|
if leaves != 2 {
|
|
t.Fatalf("expected 2 leaves, got %d", leaves)
|
|
}
|
|
// Root(InternalNode) + leaf1 (from left StemNode) + leaf2 (from right StemNode) = 3
|
|
// StemNodes are not returned as separate steps; the iterator advances
|
|
// directly to the first non-nil value within the stem.
|
|
if total != 3 {
|
|
t.Fatalf("expected 3 total nodes, got %d", total)
|
|
}
|
|
}
|