core/state/snapshot: initial wip on trie generators

2026-08-20 10:52:25 +00:00 · 2020-02-04 17:36:37 +01:00 · 2020-02-04 17:36:37 +01:00 · c2278227fa
commit c2278227fa
parent 613af7ceea
3 changed files with 306 additions and 0 deletions
--- a/core/state/snapshot/hextrie_generator.go
+++ b/core/state/snapshot/hextrie_generator.go
@ -0,0 +1,53 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package snapshot
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+type leaf struct {
+	key   common.Hash
+	value []byte
+}
+
+// trieGenerator is a very basic hexary trie builder which uses the same Trie
+// as the rest of geth, with no enhancements or optimizations
+type trieGenerator struct{}
+
+//BenchmarkTrieGeneration/4K-6         	      94	  12598506 ns/op	 6162370 B/op	   57921 allocs/op
+//BenchmarkTrieGeneration/10K-6        	      37	  33790908 ns/op	17278751 B/op	  151002 allocs/op
+func (gen *trieGenerator) Generate2(in chan (leaf), out chan (common.Hash)) {
+	t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New()))
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}
+
+
+//BenchmarkTrieGeneration/4K-6         	     115	  12755614 ns/op	 2303051 B/op	   42678 allocs/op
+//BenchmarkTrieGeneration/10K-6        	      46	  25374595 ns/op	 5754446 B/op	  106676 allocs/op
+func (gen *trieGenerator) Generate(in chan (leaf), out chan (common.Hash)) {
+	t := trie.NewAppendOnlyTrie()
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}
--- a/core/state/snapshot/trie_generator_test.go
+++ b/core/state/snapshot/trie_generator_test.go
@ -0,0 +1,154 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package snapshot
+
+import (
+	"encoding/binary"
+	"sync"
+	"testing"
+
+	"github.com/VictoriaMetrics/fastcache"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+)
+
+func generateTrie(it AccountIterator, generator *trieGenerator) common.Hash {
+	var (
+		in  = make(chan leaf)        // chan to pass leafs
+		out = make(chan common.Hash) // chan to collect result
+		wg  sync.WaitGroup
+	)
+	wg.Add(1)
+	go func() {
+		generator.Generate2(in, out)
+		wg.Done()
+	}()
+	// Feed leafs
+	for it.Next() {
+		in <- leaf{it.Hash(), it.Account()}
+	}
+	close(in)
+	result := <-out
+	wg.Wait()
+	return result
+
+}
+
+func TestTrieGeneration(t *testing.T) {
+	// Create an empty base layer and a snapshot tree out of it
+	base := &diskLayer{
+		diskdb: rawdb.NewMemoryDatabase(),
+		root:   common.HexToHash("0x01"),
+		cache:  fastcache.New(1024 * 500),
+	}
+	snaps := &Tree{
+		layers: map[common.Hash]snapshot{
+			base.root: base,
+		},
+	}
+	// Stack three diff layers on top with various overlaps
+	snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"),
+		randomAccountSet("0x11", "0x22", "0x33"), nil)
+	// We call this once before the benchmark, so the creation of
+	// sorted accountlists are not included in the results.
+	head := snaps.Snapshot(common.HexToHash("0x02"))
+	it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+	generator := &trieGenerator{}
+	hash := generateTrie(it, generator)
+	if exp, got := hash, common.HexToHash("807fbe7d4e4c62b80b1e7f682bb13ed409467df2a5903e5af44b88f6b08d0519"); exp != got {
+		t.Fatalf("expected %v got %v", exp, got)
+	}
+}
+
+func TestTrieGenerationAppendonly(t *testing.T) {
+	// Create an empty base layer and a snapshot tree out of it
+	base := &diskLayer{
+		diskdb: rawdb.NewMemoryDatabase(),
+		root:   common.HexToHash("0x01"),
+		cache:  fastcache.New(1024 * 500),
+	}
+	snaps := &Tree{
+		layers: map[common.Hash]snapshot{
+			base.root: base,
+		},
+	}
+	// Stack three diff layers on top with various overlaps
+	snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"),
+		randomAccountSet("0x11", "0x22", "0x33"), nil)
+	// We call this once before the benchmark, so the creation of
+	// sorted accountlists are not included in the results.
+	head := snaps.Snapshot(common.HexToHash("0x02"))
+	it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+	generator := &trieGenerator{}
+	hash := generateTrie(it, generator)
+	if exp, got := hash, common.HexToHash("807fbe7d4e4c62b80b1e7f682bb13ed409467df2a5903e5af44b88f6b08d0519"); exp != got {
+		t.Fatalf("expected %v got %v", exp, got)
+	}
+}
+
+func BenchmarkTrieGeneration(b *testing.B) {
+	// Get a fairly large trie
+	// Create a custom account factory to recreate the same addresses
+	makeAccounts := func(num int) map[common.Hash][]byte {
+		accounts := make(map[common.Hash][]byte)
+		for i := 0; i < num; i++ {
+			h := common.Hash{}
+			binary.BigEndian.PutUint64(h[:], uint64(i+1))
+			accounts[h] = randomAccount()
+		}
+		return accounts
+	}
+	// Build up a large stack of snapshots
+	base := &diskLayer{
+		diskdb: rawdb.NewMemoryDatabase(),
+		root:   common.HexToHash("0x01"),
+		cache:  fastcache.New(1024 * 500),
+	}
+	snaps := &Tree{
+		layers: map[common.Hash]snapshot{
+			base.root: base,
+		},
+	}
+	b.Run("4K", func(b *testing.B) {
+		// 4K accounts
+		snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(4000), nil)
+		head := snaps.Snapshot(common.HexToHash("0x02"))
+		// Call it once to make it create the lists before test starts
+		head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+			generator := &trieGenerator{}
+			generateTrie(it, generator)
+		}
+	})
+	b.Run("10K", func(b *testing.B) {
+		// 4K accounts
+		snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(10000), nil)
+		head := snaps.Snapshot(common.HexToHash("0x02"))
+		// Call it once to make it create the lists before test starts
+		head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00"))
+			generator := &trieGenerator{}
+			generateTrie(it, generator)
+		}
+	})
+}
--- a/trie/appendtrie.go
+++ b/trie/appendtrie.go
@ -0,0 +1,99 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"fmt"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// AppendOnlyTrie is a Merkle Patricia Trie, which can only be used for
+// constructing a trie from a sequence of sorted leafs, in descending order
+type AppendOnlyTrie struct {
+	root node
+}
+
+func NewAppendOnlyTrie() *AppendOnlyTrie {
+	return &AppendOnlyTrie{root:nil}
+}
+
+func (t *AppendOnlyTrie) TryUpdate(key, value []byte) error {
+	k := keybytesToHex(key)
+	if len(value) == 0 {
+		panic("deletion not supported")
+	}
+	t.root = t.insert(t.root, nil, k, valueNode(value))
+	return nil
+}
+
+func (t *AppendOnlyTrie) insert(n node, prefix, key []byte, value node) node {
+	if len(key) == 0 {
+		return value
+	}
+	switch n := n.(type) {
+	case *shortNode:
+		matchlen := prefixLen(key, n.Key)
+		// If the whole key matches, it already exists
+		if matchlen == len(n.Key) {
+			n.Val = t.insert(n.Val, append(prefix, key[:matchlen]...), key[matchlen:], value)
+			n.flags = nodeFlag{dirty: true}
+			return n
+		}
+		// Otherwise branch out at the index where they differ.
+		branch := &fullNode{flags: nodeFlag{dirty: true}}
+		branch.Children[n.Key[matchlen]]= t.insert(nil, append(prefix, n.Key[:matchlen+1]...), n.Key[matchlen+1:], n.Val)
+		// TODO: We can now shoot off n.Val for hashing
+		branch.Children[key[matchlen]]= t.insert(nil, append(prefix, key[:matchlen+1]...), key[matchlen+1:], value)
+
+		// Replace this shortNode with the branch if it occurs at index 0.
+		if matchlen == 0 {
+			return branch
+		}
+		// Otherwise, replace it with a short node leading up to the branch.
+		n.Key = key[:matchlen]
+		n.Val = branch
+		n.flags = nodeFlag{dirty: true}
+		return n
+
+	case *fullNode:
+		n.flags = nodeFlag{dirty: true}
+		n.Children[key[0]] = t.insert(n.Children[key[0]], append(prefix, key[0]), key[1:], value)
+		return n
+
+	case nil:
+		return &shortNode{key, value, nodeFlag{dirty: true}}
+
+	case hashNode:
+		// We've hit a part of the trie that isn't loaded yet -- this means
+		// someone inserted
+		panic("hash resolution not supported")
+
+	default:
+		panic(fmt.Sprintf("%T: invalid node: %v", n, n))
+	}
+}
+
+func (t *AppendOnlyTrie) Hash() common.Hash {
+	if t.root == nil {
+		return emptyRoot
+	}
+	h := newHasher(false)
+	defer returnHasherToPool(h)
+	hashed, cached := h.hash(t.root, true)
+	t.root = cached
+	return common.BytesToHash(hashed.(hashNode))
+}