From c2c17966a43fae3cc29b6022e715942042683a37 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 7 Jan 2020 14:04:57 +0100 Subject: [PATCH] trie: deprecate hasher.go, make proof framework use new hasher --- trie/hasher.go | 199 ----------------------------------------- trie/iterator.go | 8 +- trie/proof.go | 26 +++--- trie/pure_committer.go | 110 ----------------------- trie/pure_hasher.go | 18 ++++ 5 files changed, 33 insertions(+), 328 deletions(-) delete mode 100644 trie/hasher.go diff --git a/trie/hasher.go b/trie/hasher.go deleted file mode 100644 index 110ab36e2a..0000000000 --- a/trie/hasher.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" -) - -// @deprecated -// hasher is the old hash+commit utility, replaced by dedicated -// hasher (pure_hasher) and committer (pure_commit) -type hasher struct { - tmp sliceBuffer - sha keccakState - onleaf LeafCallback -} - -// hashers live in a global db. -var hasherPool = sync.Pool{ - New: func() interface{} { - return &hasher{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. - sha: sha3.NewLegacyKeccak256().(keccakState), - } - }, -} - -func newHasher(onleaf LeafCallback) *hasher { - h := hasherPool.Get().(*hasher) - h.onleaf = onleaf - return h -} - -func returnHasherToPool(h *hasher) { - h.onleaf = nil - hasherPool.Put(h) -} - -// hash collapses a node down into a hash node, also returning a copy of the -// original node initialized with the computed hash to replace the original one. -func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { - // If we're not storing the node, just hashing, use available cached data - if hash, dirty := n.cache(); hash != nil { - if db == nil { - return hash, n, nil - } - if !dirty { - switch n.(type) { - case *fullNode, *shortNode: - return hash, hash, nil - default: - return hash, n, nil - } - } - } - // Trie not processed yet or needs storage, walk the children - collapsed, cached, err := h.hashChildren(n, db) - if err != nil { - return hashNode{}, n, err - } - hashed, err := h.store(collapsed, db, force) - if err != nil { - return hashNode{}, n, err - } - // Cache the hash of the node for later reuse and remove - // the dirty flag in commit mode. It's fine to assign these values directly - // without copying the node first because hashChildren copies it. - cachedHash, _ := hashed.(hashNode) - switch cn := cached.(type) { - case *shortNode: - cn.flags.hash = cachedHash - if db != nil { - cn.flags.dirty = false - } - case *fullNode: - cn.flags.hash = cachedHash - if db != nil { - cn.flags.dirty = false - } - } - return hashed, cached, nil -} - -// hashChildren replaces the children of a node with their hashes if the encoded -// size of the child is larger than a hash, returning the collapsed node as well -// as a replacement for the original node with the child hashes cached in. -func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { - var err error - - switch n := original.(type) { - case *shortNode: - // Hash the short node's child, caching the newly hashed subtree - collapsed, cached := n.copy(), n.copy() - collapsed.Key = hexToCompact(n.Key) - cached.Key = common.CopyBytes(n.Key) - - if _, ok := n.Val.(valueNode); !ok { - collapsed.Val, cached.Val, err = h.hash(n.Val, db, false) - if err != nil { - return original, original, err - } - } - return collapsed, cached, nil - - case *fullNode: - // Hash the full node's children, caching the newly hashed subtrees - collapsed, cached := n.copy(), n.copy() - - for i := 0; i < 16; i++ { - if n.Children[i] != nil { - collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) - if err != nil { - return original, original, err - } - } - } - cached.Children[16] = n.Children[16] - return collapsed, cached, nil - - default: - // Value and hash nodes don't have children so they're left as were - return n, original, nil - } -} - -// store hashes the node n and if we have a storage layer specified, it writes -// the key/value pair to it and tracks any node->child references as well as any -// node->external trie references. -func (h *hasher) store(n node, db *Database, force bool) (node, error) { - // Don't store hashes or empty nodes. - if _, isHash := n.(hashNode); n == nil || isHash { - return n, nil - } - // Generate the RLP encoding of the node - h.tmp.Reset() - if err := rlp.Encode(&h.tmp, n); err != nil { - panic("encode error: " + err.Error()) - } - if len(h.tmp) < 32 && !force { - return n, nil // Nodes smaller than 32 bytes are stored inside their parent - } - // Larger nodes are replaced by their hash and stored in the database. - hash, _ := n.cache() - if hash == nil { - hash = h.makeHashNode(h.tmp) - } - - if db != nil { - // We are pooling the trie nodes into an intermediate memory cache - hash := common.BytesToHash(hash) - - db.lock.Lock() - db.insert(hash, len(h.tmp), n) - db.lock.Unlock() - - // Track external references from account->storage trie - if h.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - h.onleaf(child, hash) - } - case *fullNode: - for i := 0; i < 16; i++ { - if child, ok := n.Children[i].(valueNode); ok { - h.onleaf(child, hash) - } - } - } - } - } - return hash, nil -} - -func (h *hasher) makeHashNode(data []byte) hashNode { - n := make(hashNode, h.sha.Size()) - h.sha.Reset() - h.sha.Write(data) - h.sha.Read(n) - return n -} diff --git a/trie/iterator.go b/trie/iterator.go index 8e84dee3b6..f7a42572ce 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -182,15 +182,13 @@ func (it *nodeIterator) LeafBlob() []byte { func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - hasher := newHasher(nil) - defer returnHasherToPool(hasher) - + hasher := newPureHasher() + defer returnPureHasherToPool(hasher) proofs := make([][]byte, 0, len(it.stack)) for i, item := range it.stack[:len(it.stack)-1] { // Gather nodes that end up as hash nodes (or the root) - node, _, _ := hasher.hashChildren(item.node, nil) - hashed, _ := hasher.store(node, nil, false) + node, hashed := hasher.proofHash(item.node) if _, ok := hashed.(hashNode); ok || i == 0 { enc, _ := rlp.EncodeToBytes(node) proofs = append(proofs, enc) diff --git a/trie/proof.go b/trie/proof.go index 9985e730dd..bf924a08fd 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -64,26 +64,24 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - hasher := newHasher(nil) - defer returnHasherToPool(hasher) + hasher := newPureHasher() + defer returnPureHasherToPool(hasher) for i, n := range nodes { - // Don't bother checking for errors here since hasher panics - // if encoding doesn't work and we're not writing to any database. - n, _, _ = hasher.hashChildren(n, nil) - hn, _ := hasher.store(n, nil, false) + if fromLevel > 0 { + fromLevel-- + continue + } + var hn = n + n,hn = hasher.proofHash(n) if hash, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. - if fromLevel > 0 { - fromLevel-- - } else { - enc, _ := rlp.EncodeToBytes(n) - if !ok { - hash = hasher.makeHashNode(enc) - } - proofDb.Put(hash, enc) + enc, _ := rlp.EncodeToBytes(n) + if !ok { + hash = hasher.hashData(enc) } + proofDb.Put(hash, enc) } } return nil diff --git a/trie/pure_committer.go b/trie/pure_committer.go index b0e56a92f4..a5dba1a729 100644 --- a/trie/pure_committer.go +++ b/trie/pure_committer.go @@ -246,7 +246,6 @@ func (h *committer) commitLoop(db *Database) { } func (h *committer) makeHashNode(data []byte) hashNode { - //fmt.Printf("hashing: %x\n", data) n := make(hashNode, h.sha.Size()) h.sha.Reset() h.sha.Write(data) @@ -283,112 +282,3 @@ func estimateSize(n node) int { } } - -/** -Todo, we could improve the situation for small trie commits (storage tries), -if we use one dedicated database-inserter, instead of having each one spin up a -separate instance. - -The gain is not only that we save some goroutine start/stop, it's also that -we can process trie M while we're still committing trie N -- since we don't -have to do the waitgroup-wait between each trie commit. - -The code below is a rough sketch, it needs to be integrated nicely without causing -dependency cycles between state, core and trie. - - - -type DbInserter struct { - inputCh chan *Leaf // This is where input to database is sent - reportCh chan int // At certain points, callers wants to know that we're done - db *Database - wg sync.WaitGroup -} - -// commitLoop does the actual insert + leaf callback for nodes -func (dbi *DbInserter) run() { - defer dbi.wg.Done() - for item := range dbi.inputCh { - var ( - hash = item.hash - size = item.size - n = item.node - hasVnodes = item.vnodes - onleaf = item.onLeaf - ) - if size < 0 { - // This is an end-marker object. - dbi.reportCh <- size - continue - } - // We are pooling the trie nodes into an intermediate memory cache - dbi.db.lock.Lock() - dbi.db.insert(hash, size, n) - dbi.db.lock.Unlock() - if onleaf != nil && hasVnodes { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - onleaf(child, hash) - } - case *fullNode: - for i := 0; i < 16; i++ { - if child, ok := n.Children[i].(valueNode); ok { - onleaf(child, hash) - } - } - } - } - } -} - -func (dbi *DbInserter) Close() { - close(dbi.inputCh) - dbi.wg.Wait() -} - -func (dbi *DbInserter) Insert(leaf *Leaf) { - dbi.inputCh <- leaf -} - -// WaitForEmpty returns to the caller when all the data currently in the -// channel has been handled -func (dbi *DbInserter) WaitForEmpty() { - // Send an arbitrary id there - checksum := rand.Uint32() - dbi.inputCh <- &trie.Leaf{ - size: -checksum, - } - // And wait for it to come back - for { - select { - case retval <- dbi.reportCh: - if retval == checksum { - return - } - - } - } -} - -func (dbi *DbInserter) InsertBlob(blob []byte, blobHash common.Hash) { - dbi.inputCh <- &trie.Leaf{ - size: len(blob), - hash: blobHash, - node: rawNode(blob), - vnodes: false, - } -} - -func StartDBInserter(db *Database) *DbInserter { - - dbi := &DbInserter{ - inputCh: make(chan *Leaf, 200), - reportCh: make(chan int), - db: db, - } - go dbi.run() -} - - -*/ diff --git a/trie/pure_hasher.go b/trie/pure_hasher.go index facfbe1588..5896a5ed49 100644 --- a/trie/pure_hasher.go +++ b/trie/pure_hasher.go @@ -179,3 +179,21 @@ func (h *pureHasher) hashData(data []byte) hashNode { h.sha.Read(n) return n } + +// proofHash is used to construct trie proofs, and returns the 'collapsed' +// node (for later RLP encoding) aswell as the hashed node -- unless the +// node is smaller than 32 bytes, in which case it will be returned as is. +// This method does not do anything on value- or hash-nodes. +func (h *pureHasher) proofHash(original node) (collapsed, hashed node){ + switch n := original.(type) { + case *shortNode: + sn,_ := h.hashShortNodeChildren(n) + return sn, h.shortnodeToHash(sn, false) + case *fullNode: + fn,_ := h.hashFullNodeChildren(n) + return fn, h.fullnodeToHash(fn, false) + default: + // Value and hash nodes don't have children so they're left as were + return n, n + } +} \ No newline at end of file