diff --git a/core/state/snapshot/hextrie_generator.go b/core/state/snapshot/hextrie_generator.go new file mode 100644 index 0000000000..60cd291605 --- /dev/null +++ b/core/state/snapshot/hextrie_generator.go @@ -0,0 +1,53 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/trie" +) + +type leaf struct { + key common.Hash + value []byte +} + +// trieGenerator is a very basic hexary trie builder which uses the same Trie +// as the rest of geth, with no enhancements or optimizations +type trieGenerator struct{} + +//BenchmarkTrieGeneration/4K-6 94 12598506 ns/op 6162370 B/op 57921 allocs/op +//BenchmarkTrieGeneration/10K-6 37 33790908 ns/op 17278751 B/op 151002 allocs/op +func (gen *trieGenerator) Generate2(in chan (leaf), out chan (common.Hash)) { + t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New())) + for leaf := range in { + t.TryUpdate(leaf.key[:], leaf.value) + } + out <- t.Hash() +} + + +//BenchmarkTrieGeneration/4K-6 115 12755614 ns/op 2303051 B/op 42678 allocs/op +//BenchmarkTrieGeneration/10K-6 46 25374595 ns/op 5754446 B/op 106676 allocs/op +func (gen *trieGenerator) Generate(in chan (leaf), out chan (common.Hash)) { + t := trie.NewAppendOnlyTrie() + for leaf := range in { + t.TryUpdate(leaf.key[:], leaf.value) + } + out <- t.Hash() +} diff --git a/core/state/snapshot/trie_generator_test.go b/core/state/snapshot/trie_generator_test.go new file mode 100644 index 0000000000..cc24878adb --- /dev/null +++ b/core/state/snapshot/trie_generator_test.go @@ -0,0 +1,154 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snapshot + +import ( + "encoding/binary" + "sync" + "testing" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" +) + +func generateTrie(it AccountIterator, generator *trieGenerator) common.Hash { + var ( + in = make(chan leaf) // chan to pass leafs + out = make(chan common.Hash) // chan to collect result + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + generator.Generate2(in, out) + wg.Done() + }() + // Feed leafs + for it.Next() { + in <- leaf{it.Hash(), it.Account()} + } + close(in) + result := <-out + wg.Wait() + return result + +} + +func TestTrieGeneration(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), + randomAccountSet("0x11", "0x22", "0x33"), nil) + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := snaps.Snapshot(common.HexToHash("0x02")) + it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + generator := &trieGenerator{} + hash := generateTrie(it, generator) + if exp, got := hash, common.HexToHash("807fbe7d4e4c62b80b1e7f682bb13ed409467df2a5903e5af44b88f6b08d0519"); exp != got { + t.Fatalf("expected %v got %v", exp, got) + } +} + +func TestTrieGenerationAppendonly(t *testing.T) { + // Create an empty base layer and a snapshot tree out of it + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + // Stack three diff layers on top with various overlaps + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), + randomAccountSet("0x11", "0x22", "0x33"), nil) + // We call this once before the benchmark, so the creation of + // sorted accountlists are not included in the results. + head := snaps.Snapshot(common.HexToHash("0x02")) + it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + generator := &trieGenerator{} + hash := generateTrie(it, generator) + if exp, got := hash, common.HexToHash("807fbe7d4e4c62b80b1e7f682bb13ed409467df2a5903e5af44b88f6b08d0519"); exp != got { + t.Fatalf("expected %v got %v", exp, got) + } +} + +func BenchmarkTrieGeneration(b *testing.B) { + // Get a fairly large trie + // Create a custom account factory to recreate the same addresses + makeAccounts := func(num int) map[common.Hash][]byte { + accounts := make(map[common.Hash][]byte) + for i := 0; i < num; i++ { + h := common.Hash{} + binary.BigEndian.PutUint64(h[:], uint64(i+1)) + accounts[h] = randomAccount() + } + return accounts + } + // Build up a large stack of snapshots + base := &diskLayer{ + diskdb: rawdb.NewMemoryDatabase(), + root: common.HexToHash("0x01"), + cache: fastcache.New(1024 * 500), + } + snaps := &Tree{ + layers: map[common.Hash]snapshot{ + base.root: base, + }, + } + b.Run("4K", func(b *testing.B) { + // 4K accounts + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(4000), nil) + head := snaps.Snapshot(common.HexToHash("0x02")) + // Call it once to make it create the lists before test starts + head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + generator := &trieGenerator{} + generateTrie(it, generator) + } + }) + b.Run("10K", func(b *testing.B) { + // 4K accounts + snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), makeAccounts(10000), nil) + head := snaps.Snapshot(common.HexToHash("0x02")) + // Call it once to make it create the lists before test starts + head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + it := head.(*diffLayer).AccountIterator(common.HexToHash("0x00")) + generator := &trieGenerator{} + generateTrie(it, generator) + } + }) +} diff --git a/trie/appendtrie.go b/trie/appendtrie.go new file mode 100644 index 0000000000..f57acf58eb --- /dev/null +++ b/trie/appendtrie.go @@ -0,0 +1,99 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" +) + +// AppendOnlyTrie is a Merkle Patricia Trie, which can only be used for +// constructing a trie from a sequence of sorted leafs, in descending order +type AppendOnlyTrie struct { + root node +} + +func NewAppendOnlyTrie() *AppendOnlyTrie { + return &AppendOnlyTrie{root:nil} +} + +func (t *AppendOnlyTrie) TryUpdate(key, value []byte) error { + k := keybytesToHex(key) + if len(value) == 0 { + panic("deletion not supported") + } + t.root = t.insert(t.root, nil, k, valueNode(value)) + return nil +} + +func (t *AppendOnlyTrie) insert(n node, prefix, key []byte, value node) node { + if len(key) == 0 { + return value + } + switch n := n.(type) { + case *shortNode: + matchlen := prefixLen(key, n.Key) + // If the whole key matches, it already exists + if matchlen == len(n.Key) { + n.Val = t.insert(n.Val, append(prefix, key[:matchlen]...), key[matchlen:], value) + n.flags = nodeFlag{dirty: true} + return n + } + // Otherwise branch out at the index where they differ. + branch := &fullNode{flags: nodeFlag{dirty: true}} + branch.Children[n.Key[matchlen]]= t.insert(nil, append(prefix, n.Key[:matchlen+1]...), n.Key[matchlen+1:], n.Val) + // TODO: We can now shoot off n.Val for hashing + branch.Children[key[matchlen]]= t.insert(nil, append(prefix, key[:matchlen+1]...), key[matchlen+1:], value) + + // Replace this shortNode with the branch if it occurs at index 0. + if matchlen == 0 { + return branch + } + // Otherwise, replace it with a short node leading up to the branch. + n.Key = key[:matchlen] + n.Val = branch + n.flags = nodeFlag{dirty: true} + return n + + case *fullNode: + n.flags = nodeFlag{dirty: true} + n.Children[key[0]] = t.insert(n.Children[key[0]], append(prefix, key[0]), key[1:], value) + return n + + case nil: + return &shortNode{key, value, nodeFlag{dirty: true}} + + case hashNode: + // We've hit a part of the trie that isn't loaded yet -- this means + // someone inserted + panic("hash resolution not supported") + + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +func (t *AppendOnlyTrie) Hash() common.Hash { + if t.root == nil { + return emptyRoot + } + h := newHasher(false) + defer returnHasherToPool(h) + hashed, cached := h.hash(t.root, true) + t.root = cached + return common.BytesToHash(hashed.(hashNode)) +} \ No newline at end of file