From 8f6e4e972d02baa72a0ac84f132f34295ba8e959 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Thu, 18 Jan 2018 02:19:24 +0100 Subject: [PATCH] docs and cleanup --- core/blockchain.go | 5 ++- core/hashtree/gc.go | 70 ++++++++++++++++++++++++++------------- core/hashtree/hashtree.go | 31 +++++++++++++++++ core/state/database.go | 13 +++++++- core/state/statedb.go | 17 ++++++++-- trie/encoding.go | 24 +++++++++----- trie/trie.go | 16 +++++++-- 7 files changed, 137 insertions(+), 39 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 1759b6587a..f71e5c6da2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -181,7 +181,10 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co bc.gc.FullGC(headBlock - 1000) }*/ - bc.gc.BackgroundGC(bc.CurrentBlock, &bc.processing, &bc.procInterrupt, &bc.wg) + currentVersion := func() uint64 { + return bc.CurrentBlock().NumberU64() + } + bc.gc.BackgroundGC(currentVersion, &bc.processing, &bc.procInterrupt, &bc.wg) // Take ownership of this particular state go bc.update() diff --git a/core/hashtree/gc.go b/core/hashtree/gc.go index ad3a069459..8b74c5a2e9 100644 --- a/core/hashtree/gc.go +++ b/core/hashtree/gc.go @@ -25,13 +25,13 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/syndtr/goleveldb/leveldb/util" ) +// Print is a debug tool that dumps the contents of the database under a certain prefix func Print(db ethdb.Database, prefix []byte) { it := db.(*ethdb.LDBDatabase).NewIterator() defer it.Release() @@ -47,20 +47,23 @@ func Print(db ethdb.Database, prefix []byte) { } } -type hasDataFn func(version uint64) func(position, hash []byte) bool +// hasDataFn callback is required for garbage collecting a data structure. It returns +// another callback for each actual GC version that tells the GC whether a given element +// is present in that version of the structure at the given position. +type hasDataFn func(gcVersion uint64) func(position, hash []byte) bool type GarbageCollector struct { db *ethdb.LDBDatabase prefix []byte hasData hasDataFn - gcBlock uint64 - gcBlockHasData func(position, hash []byte) bool + gcVersion uint64 + gcVersionHasData func(position, hash []byte) bool delkeys [][]byte keysChecked, keysRemoved uint64 refsChecked, refsRemoved uint64 writeCounter uint64 writeLock sync.Mutex - valid bool + dbWrite bool } func NewGarbageCollector(db ethdb.Database, prefix []byte, hasData hasDataFn) *GarbageCollector { @@ -71,9 +74,18 @@ func NewGarbageCollector(db ethdb.Database, prefix []byte, hasData hasDataFn) *G } } +// run iterates through a section of the database and deletes old entries. First only the reference +// entries are deleted, data entries are only marked for deletion. +// +// Note: writeLock is not held while collecting entries for deletion because that would hurt block +// processing performance. Instead, dbWrite flag shows if new entries were added to the database +// while collecting data entries to be deleted. In this case, to avoid a race condition, data entries +// are not deleted because they might have been recently added again with new references. The inclusion +// checking effort is not lost though, when GC arrives there again in the next round, these data +// entries are immediately deleted without any further checks if no new references have been added. func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []byte) { g.writeLock.Lock() - g.valid = true + g.dbWrite = false g.writeLock.Unlock() it := g.db.NewIterator() @@ -82,7 +94,7 @@ func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []by defer func() { it.Release() g.writeLock.Lock() - if g.valid { + if !g.dbWrite { for _, key := range g.delkeys { g.db.Delete(key) } @@ -99,7 +111,7 @@ func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []by g.db.LDB().CompactRange(r) }() - g.gcBlockHasData = g.hasData(g.gcBlock) + g.gcVersionHasData = g.hasData(g.gcVersion) it.Seek(startKey) for it.Valid() { key := common.CopyBytes(it.Key()) @@ -145,7 +157,7 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) { oldrefs := 0 for oldrefs < refcount { version := binary.BigEndian.Uint64(refkeys[oldrefs][keylen-1 : keylen+7]) - if version >= g.gcBlock { + if version >= g.gcVersion { break } oldrefs++ @@ -154,7 +166,7 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) { removerefs := 0 if oldrefs > 0 { removerefs = oldrefs - 1 - if oldrefs == refcount && !g.gcBlockHasData(key[len(g.prefix):keylen-33], key[keylen-33:keylen-1]) { + if oldrefs == refcount && !g.gcVersionHasData(key[len(g.prefix):keylen-33], key[keylen-33:keylen-1]) { removerefs = refcount } } @@ -170,22 +182,28 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) { } } -func (g *GarbageCollector) FullGC(block uint64) { - log.Info("Starting full GC", "block", block) - g.gcBlock = block +// FullGC iterates through the entire database and removes all garbage +func (g *GarbageCollector) FullGC(version uint64) { + log.Info("Starting full GC", "version", version) + g.gcVersion = version key := g.prefix for key != nil { key = g.run(key, 10000) - k := key + k := key[len(g.prefix):] if len(k) > 8 { k = k[:8] } - log.Info("Running...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved) + log.Info("Running...", "key", fmt.Sprintf("%016x", k), "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved) } log.Info("Finished full GC", "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved) } -func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, processing, stop *int32, wg *sync.WaitGroup) { +// BackgroundGC runs in the background while stop is 0 and starts a GC for the next short section of the database +// when writeCounter has been increased enough by a Writer and pause is also 0. +// +// Note: pause does not guarantee anything but can be used to usually avoid collision between writes and GC deletions +// and thereby increase the performance of both processes. +func (g *GarbageCollector) BackgroundGC(currentVersion func() uint64, pause, stop *int32, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() @@ -200,20 +218,23 @@ func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, proces gcCounter = wc - 10000 diff = 10000 } - if diff >= 100 && atomic.LoadInt32(processing) == 0 { + if diff >= 100 && atomic.LoadInt32(pause) == 0 { gcCounter += 100 if key == nil { key = g.prefix } - headBlock := currentBlock().NumberU64() - if headBlock > 1000 { - g.gcBlock = headBlock - 1000 + headVersion := currentVersion() + if headVersion > 1000 { + g.gcVersion = headVersion - 1000 key = g.run(key, 1000) - k := key + if key == nil { + key = g.prefix + } + k := key[len(g.prefix):] if len(k) > 8 { k = k[:8] } - log.Info("Running GC...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved) + log.Info("Running GC...", "key", fmt.Sprintf("%016x", k), "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved) } } else { time.Sleep(time.Second) @@ -222,11 +243,14 @@ func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, proces }() } +// LockWrite should be called before writing to the backing database. If a Writer is used with a batch of the +// backing database then it should be called before committing the batch. func (g *GarbageCollector) LockWrite() { g.writeLock.Lock() - g.valid = false + g.dbWrite = true } +// UnlockWrite should be called after writing to the backing database func (g *GarbageCollector) UnlockWrite() { g.writeLock.Unlock() } diff --git a/core/hashtree/hashtree.go b/core/hashtree/hashtree.go index 8fcf2c49bd..8f5c9edb3e 100644 --- a/core/hashtree/hashtree.go +++ b/core/hashtree/hashtree.go @@ -16,6 +16,32 @@ package hashtree +// Package hashtree defines a general storage model for evolving tree-hashed data +// structures and implements garbage collection that removes elements which were only +// referenced by old versions of the structure that are no longer necessary to store. +// +// The storage model requires a definition of the data structure that assigns a position +// to each hashed element. The format of the position is defined by the data structure. +// A function is required that can tell for each (version, position, hash) tuple whether +// the given hashed element is part of the given version of the structure at the given +// position. +// +// Each version of the structure is identified by its root hash and also has a version +// number. Garbage collection can delete all elements that are only referenced in versions +// with a version number lower than a certain value ("GC version"). The evolution of the +// structure can be rolled back and version numbers can be reused but no rollback is +// allowed at or below the GC version. +// +// When writing a new version to the hash tree storage, each element not present in its +// parent version has to be written with the new version number. Elements are stored in +// the backing database in the following format: +// +// position + hash + []byte{0} -> data +// position + hash + version (uint64 big endian) -> NULL +// +// Reads only access the data entry, write operations always add the later (reference) +// entry too. + import ( "encoding/binary" "sync/atomic" @@ -31,6 +57,7 @@ type DatabaseWriter interface { Put([]byte, []byte) error } +// Reader provides read access to the hash tree storage type Reader struct { db DatabaseReader prefix []byte @@ -41,6 +68,7 @@ func NewReader(db DatabaseReader, prefix string) *Reader { return &Reader{db, []byte(prefix), len(prefix)} } +// Get returns elements by position and hash func (h *Reader) Get(position, hash []byte) ([]byte, error) { lp, lh := len(position), len(hash) key := make([]byte, h.lpf+lp+lh+1) @@ -64,10 +92,12 @@ func (h *Reader) Has(position, hash []byte) (bool, error) { return h.db.Has(key) } +// Put should never be used, Reader still implements r/w database interfaces for convenient use with tries func (h *Reader) Put(position, hash, data []byte) error { panic(nil) } +// Writer provides write access to the hash tree storage. A new writer is required for each new version. type Writer struct { db DatabaseWriter prefix []byte @@ -89,6 +119,7 @@ func NewWriter(db DatabaseWriter, prefix string, version uint64, gc *GarbageColl return w } +// Put adds an element and a version reference entry to the hash tree func (w *Writer) Put(position, hash, data []byte) error { if w.gc != nil { atomic.AddUint64(&w.gc.writeCounter, 1) diff --git a/core/state/database.go b/core/state/database.go index 9d64b4fcd0..1eff70aeb4 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -155,6 +155,15 @@ func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) { return root, err } +const ( + htContractCodeSuffix = 5 + htContractStorageSuffix = 6 +) + +// storageTrieDb implements trie.Database for contract storage tries +// +// a contract storage trie node's hash tree position is encoded as: +// contractAddressHash + htContractStorageSuffix + storage trie node position type storageTrieDb struct { dbr trie.DatabaseReader dbw trie.DatabaseWriter @@ -164,7 +173,7 @@ type storageTrieDb struct { func (s *storageTrieDb) position(position []byte) []byte { pos := make([]byte, len(position)+33) copy(pos[:32], s.addrHash) - pos[32] = 6 + pos[32] = htContractStorageSuffix copy(pos[33:], position) return pos } @@ -182,6 +191,8 @@ func (s *storageTrieDb) Has(position, hash []byte) (bool, error) { return s.dbr.Has(s.position(position), hash) } +// a contract code's hash tree position is encoded as: +// contractAddressHash + htContractCodeSuffix func contractCodePosition(addrHash common.Hash) []byte { return append(addrHash.Bytes(), 5) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 676a68ee9e..7d63e2387b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -606,6 +606,11 @@ func (s *StateDB) CommitTo(db hashtree.DatabaseWriter, blockNumber uint64, gc *h return root, err } +// HasDataCallback returns a GC callback function for a given state (identified by state root). +// The callback tells if any state trie node, secure trie key preimage, contract code or contract storage +// trie node is part of the given trie at the given position. The hash tree position encoding of state +// trie nodes is identical to the general trie node position encoding. For contract code and storage +// position encoding see contractCodePosition and storageTrieDb. func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(position, hash []byte) bool { db := hashtree.NewReader(dbr, DbPrefix) t, err := trie.New(root, db) @@ -614,9 +619,13 @@ func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(positio } return func(position, hash []byte) bool { lp := len(position) - if lp < 33 || (lp == 33 && position[32] < 5) { + if lp < 33 || (lp == 33 && position[32] < htContractCodeSuffix) { + // it should be a state trie node, check it there return t.HasData(position, hash) } + // it it either a code or a storage trie node, in either case we need the + // account entry to check. We do this manually with a "regular" (not secure) + // trie because we only know the address hash addrHash := position[:32] enc, err := t.TryGet(addrHash) if len(enc) == 0 || err != nil { @@ -627,13 +636,15 @@ func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(positio if err := rlp.DecodeBytes(enc, &data); err != nil { return false } - if lp == 33 && position[32] == 5 { + if lp == 33 && position[32] == htContractCodeSuffix { + // if it is a code, the hash should match the currently present account's code hash return bytes.Equal(hash, data.CodeHash) } - if position[32] != 6 { + if position[32] != htContractStorageSuffix { return false } + // it is a storage trie node, check in the storage trie st, err := trie.New(data.Root, &storageTrieDb{dbr: db, addrHash: addrHash}) if err != nil { return false diff --git a/trie/encoding.go b/trie/encoding.go index 7be88ff9c1..ff9fa6663f 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -113,15 +113,24 @@ func hasTerm(s []byte) bool { return len(s) > 0 && s[len(s)-1] == 16 } +const ( + htEvenNibbleSuffix = 0 + htOddNibbleSuffix = 1 + htSecTrieKeySuffix = 4 +) + +// hexToHashTreePos converts a hex encoded trie key prefix to a hash tree position. +// Trie hash tree position encoding +// - for even number of nibbles: nibbles[0]*16+nibbles[1], ..., nibbles[i*2]*16+nibbles[i*2+1], 0 +// - for odd number of nibbles: nibbles[0]*16+nibbles[1], ..., nibbles[i*2+1]*16+1 func hexToHashTreePos(hex []byte) []byte { - terminator := byte(0) + terminator := byte(htEvenNibbleSuffix) if hasTerm(hex) { - terminator = 2 hex = hex[:len(hex)-1] } buf := make([]byte, len(hex)/2+1) if len(hex)&1 == 1 { - terminator += hex[len(hex)-1]<<4 + 1 + terminator = hex[len(hex)-1]<<4 + htOddNibbleSuffix hex = hex[:len(hex)-1] } decodeNibbles(hex, buf[:len(buf)-1]) @@ -129,18 +138,17 @@ func hexToHashTreePos(hex []byte) []byte { return buf } +// SecHashTreePos returns the hash tree position for secure trie key preimage entries func SecHashTreePos(hash []byte) []byte { - return append(hash, 4) + return append(hash, htSecTrieKeySuffix) } +// hashTreePosToHex converts hash tree position (either of a trie node or a secure trie +// key preimage entry) back to hex encoding func hashTreePosToHex(pos []byte) []byte { base := keybytesToHex(pos) base = base[:len(base)-1] term := base[len(base)-1] base = base[:len(base)-2+int(term&1)] - // apply terminator flag - if term >= 2 { - base = append(base, 16) - } return base } diff --git a/trie/trie.go b/trie/trie.go index eddf519872..1eee48e6b9 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -504,18 +504,28 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) { return h.hash(t.root, db, nil, true) } +// HasData checks whether a trie node or the entry belonging to a secure trie key +// preimage is present in the trie func (t *Trie) HasData(position, hash []byte) bool { + secTrieKey := len(position) > 0 && position[len(position)-1] == htSecTrieKeySuffix + if secTrieKey && (len(position) != len(hash)+1 || !bytes.Equal(position[:len(hash)], hash)) { + // position for a secure trie key is always hash + htSecTrieKeySuffix + return false + } hex := hashTreePosToHex(position) //fmt.Println("pos", position, "hex", hex, "hash", hash) n, err := t.ProveHexKey(hex, 0, nil) if n == nil || err != nil { return false } + if secTrieKey { + // for secure trie keys we only care about whether the given trie contains an + // entry at that key, regardless of its contents + return true + } hasher := newHasher(0, 0) n, _, _ = hasher.hashChildren(n, nil, nil) hn, _ := hasher.store(n, nil, nil, false) nodeHash, ok := hn.(hashNode) - eq := ok && bytes.Equal(nodeHash, hash) - //fmt.Println("eq", eq, ok, nodeHash, hash) - return eq + return ok && bytes.Equal(nodeHash, hash) }