docs and cleanup

core/blockchain.go

@@ -181,7 +181,10 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
 		bc.gc.FullGC(headBlock - 1000)
 	}*/
 
-	bc.gc.BackgroundGC(bc.CurrentBlock, &bc.processing, &bc.procInterrupt, &bc.wg)
+	currentVersion := func() uint64 {
+		return bc.CurrentBlock().NumberU64()
+	}
+	bc.gc.BackgroundGC(currentVersion, &bc.processing, &bc.procInterrupt, &bc.wg)
 
 	// Take ownership of this particular state
 	go bc.update()

core/hashtree/gc.go

@@ -25,13 +25,13 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+// Print is a debug tool that dumps the contents of the database under a certain prefix
 func Print(db ethdb.Database, prefix []byte) {
 	it := db.(*ethdb.LDBDatabase).NewIterator()
 	defer it.Release()
@@ -47,20 +47,23 @@ func Print(db ethdb.Database, prefix []byte) {
 	}
 }
 
-type hasDataFn func(version uint64) func(position, hash []byte) bool
+// hasDataFn callback is required for garbage collecting a data structure. It returns
+// another callback for each actual GC version that tells the GC whether a given element
+// is present in that version of the structure at the given position.
+type hasDataFn func(gcVersion uint64) func(position, hash []byte) bool
 
 type GarbageCollector struct {
 	db                       *ethdb.LDBDatabase
 	prefix                   []byte
 	hasData                  hasDataFn
-	gcBlock                  uint64
+	gcVersion                uint64
-	gcBlockHasData           func(position, hash []byte) bool
+	gcVersionHasData         func(position, hash []byte) bool
 	delkeys                  [][]byte
 	keysChecked, keysRemoved uint64
 	refsChecked, refsRemoved uint64
 	writeCounter             uint64
 	writeLock                sync.Mutex
-	valid                    bool
+	dbWrite                  bool
 }
 
 func NewGarbageCollector(db ethdb.Database, prefix []byte, hasData hasDataFn) *GarbageCollector {
@@ -71,9 +74,18 @@ func NewGarbageCollector(db ethdb.Database, prefix []byte, hasData hasDataFn) *G
 	}
 }
 
+// run iterates through a section of the database and deletes old entries. First only the reference
+// entries are deleted, data entries are only marked for deletion.
+//
+// Note: writeLock is not held while collecting entries for deletion because that would hurt block
+// processing performance. Instead, dbWrite flag shows if new entries were added to the database
+// while collecting data entries to be deleted. In this case, to avoid a race condition, data entries
+// are not deleted because they might have been recently added again with new references. The inclusion
+// checking effort is not lost though, when GC arrives there again in the next round, these data
+// entries are immediately deleted without any further checks if no new references have been added.
 func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []byte) {
 	g.writeLock.Lock()
-	g.valid = true
+	g.dbWrite = false
 	g.writeLock.Unlock()
 
 	it := g.db.NewIterator()
@@ -82,7 +94,7 @@ func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []by
 	defer func() {
 		it.Release()
 		g.writeLock.Lock()
-		if g.valid {
+		if !g.dbWrite {
 			for _, key := range g.delkeys {
 				g.db.Delete(key)
 			}
@@ -99,7 +111,7 @@ func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []by
 		g.db.LDB().CompactRange(r)
 	}()
 
-	g.gcBlockHasData = g.hasData(g.gcBlock)
+	g.gcVersionHasData = g.hasData(g.gcVersion)
 	it.Seek(startKey)
 	for it.Valid() {
 		key := common.CopyBytes(it.Key())
@@ -145,7 +157,7 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) {
 	oldrefs := 0
 	for oldrefs < refcount {
 		version := binary.BigEndian.Uint64(refkeys[oldrefs][keylen-1 : keylen+7])
-		if version >= g.gcBlock {
+		if version >= g.gcVersion {
 			break
 		}
 		oldrefs++
@@ -154,7 +166,7 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) {
 	removerefs := 0
 	if oldrefs > 0 {
 		removerefs = oldrefs - 1
-		if oldrefs == refcount && !g.gcBlockHasData(key[len(g.prefix):keylen-33], key[keylen-33:keylen-1]) {
+		if oldrefs == refcount && !g.gcVersionHasData(key[len(g.prefix):keylen-33], key[keylen-33:keylen-1]) {
 			removerefs = refcount
 		}
 	}
@@ -170,22 +182,28 @@ func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) {
 	}
 }
 
-func (g *GarbageCollector) FullGC(block uint64) {
+// FullGC iterates through the entire database and removes all garbage
-	log.Info("Starting full GC", "block", block)
+func (g *GarbageCollector) FullGC(version uint64) {
-	g.gcBlock = block
+	log.Info("Starting full GC", "version", version)
+	g.gcVersion = version
 	key := g.prefix
 	for key != nil {
 		key = g.run(key, 10000)
-		k := key
+		k := key[len(g.prefix):]
 		if len(k) > 8 {
 			k = k[:8]
 		}
-		log.Info("Running...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
+		log.Info("Running...", "key", fmt.Sprintf("%016x", k), "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
 	}
 	log.Info("Finished full GC", "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
 }
 
-func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, processing, stop *int32, wg *sync.WaitGroup) {
+// BackgroundGC runs in the background while stop is 0 and starts a GC for the next short section of the database
+// when writeCounter has been increased enough by a Writer and pause is also 0.
+//
+// Note: pause does not guarantee anything but can be used to usually avoid collision between writes and GC deletions
+// and thereby increase the performance of both processes.
+func (g *GarbageCollector) BackgroundGC(currentVersion func() uint64, pause, stop *int32, wg *sync.WaitGroup) {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -200,20 +218,23 @@ func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, proces
 				gcCounter = wc - 10000
 				diff = 10000
 			}
-			if diff >= 100 && atomic.LoadInt32(processing) == 0 {
+			if diff >= 100 && atomic.LoadInt32(pause) == 0 {
 				gcCounter += 100
 				if key == nil {
 					key = g.prefix
 				}
-				headBlock := currentBlock().NumberU64()
+				headVersion := currentVersion()
-				if headBlock > 1000 {
+				if headVersion > 1000 {
-					g.gcBlock = headBlock - 1000
+					g.gcVersion = headVersion - 1000
 					key = g.run(key, 1000)
-					k := key
+					if key == nil {
+						key = g.prefix
+					}
+					k := key[len(g.prefix):]
 					if len(k) > 8 {
 						k = k[:8]
 					}
-					log.Info("Running GC...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
+					log.Info("Running GC...", "key", fmt.Sprintf("%016x", k), "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
 				}
 			} else {
 				time.Sleep(time.Second)
@@ -222,11 +243,14 @@ func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, proces
 	}()
 }
 
+// LockWrite should be called before writing to the backing database. If a Writer is used with a batch of the
+// backing database then it should be called before committing the batch.
 func (g *GarbageCollector) LockWrite() {
 	g.writeLock.Lock()
-	g.valid = false
+	g.dbWrite = true
 }
 
+// UnlockWrite should be called after writing to the backing database
 func (g *GarbageCollector) UnlockWrite() {
 	g.writeLock.Unlock()
 }

core/hashtree/hashtree.go

@@ -16,6 +16,32 @@
 
 package hashtree
 
+// Package hashtree defines a general storage model for evolving tree-hashed data
+// structures and implements garbage collection that removes elements which were only
+// referenced by old versions of the structure that are no longer necessary to store.
+//
+// The storage model requires a definition of the data structure that assigns a position
+// to each hashed element. The format of the position is defined by the data structure.
+// A function is required that can tell for each (version, position, hash) tuple whether
+// the given hashed element is part of the given version of the structure at the given
+// position.
+//
+// Each version of the structure is identified by its root hash and also has a version
+// number. Garbage collection can delete all elements that are only referenced in versions
+// with a version number lower than a certain value ("GC version"). The evolution of the
+// structure can be rolled back and version numbers can be reused but no rollback is
+// allowed at or below the GC version.
+//
+// When writing a new version to the hash tree storage, each element not present in its
+// parent version has to be written with the new version number. Elements are stored in
+// the backing database in the following format:
+//
+//  position + hash + []byte{0} -> data
+//  position + hash + version (uint64 big endian) -> NULL
+//
+// Reads only access the data entry, write operations always add the later (reference)
+// entry too.
+
 import (
 	"encoding/binary"
 	"sync/atomic"
@@ -31,6 +57,7 @@ type DatabaseWriter interface {
 	Put([]byte, []byte) error
 }
 
+// Reader provides read access to the hash tree storage
 type Reader struct {
 	db     DatabaseReader
 	prefix []byte
@@ -41,6 +68,7 @@ func NewReader(db DatabaseReader, prefix string) *Reader {
 	return &Reader{db, []byte(prefix), len(prefix)}
 }
 
+// Get returns elements by position and hash
 func (h *Reader) Get(position, hash []byte) ([]byte, error) {
 	lp, lh := len(position), len(hash)
 	key := make([]byte, h.lpf+lp+lh+1)
@@ -64,10 +92,12 @@ func (h *Reader) Has(position, hash []byte) (bool, error) {
 	return h.db.Has(key)
 }
 
+// Put should never be used, Reader still implements r/w database interfaces for convenient use with tries
 func (h *Reader) Put(position, hash, data []byte) error {
 	panic(nil)
 }
 
+// Writer provides write access to the hash tree storage. A new writer is required for each new version.
 type Writer struct {
 	db         DatabaseWriter
 	prefix     []byte
@@ -89,6 +119,7 @@ func NewWriter(db DatabaseWriter, prefix string, version uint64, gc *GarbageColl
 	return w
 }
 
+// Put adds an element and a version reference entry to the hash tree
 func (w *Writer) Put(position, hash, data []byte) error {
 	if w.gc != nil {
 		atomic.AddUint64(&w.gc.writeCounter, 1)

core/state/database.go

@@ -155,6 +155,15 @@ func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
 	return root, err
 }
 
+const (
+	htContractCodeSuffix    = 5
+	htContractStorageSuffix = 6
+)
+
+// storageTrieDb implements trie.Database for contract storage tries
+//
+// a contract storage trie node's hash tree position is encoded as:
+//  contractAddressHash + htContractStorageSuffix + storage trie node position
 type storageTrieDb struct {
 	dbr      trie.DatabaseReader
 	dbw      trie.DatabaseWriter
@@ -164,7 +173,7 @@ type storageTrieDb struct {
 func (s *storageTrieDb) position(position []byte) []byte {
 	pos := make([]byte, len(position)+33)
 	copy(pos[:32], s.addrHash)
-	pos[32] = 6
+	pos[32] = htContractStorageSuffix
 	copy(pos[33:], position)
 	return pos
 }
@@ -182,6 +191,8 @@ func (s *storageTrieDb) Has(position, hash []byte) (bool, error) {
 	return s.dbr.Has(s.position(position), hash)
 }
 
+// a contract code's hash tree position is encoded as:
+//  contractAddressHash + htContractCodeSuffix
 func contractCodePosition(addrHash common.Hash) []byte {
 	return append(addrHash.Bytes(), 5)
 }

core/state/statedb.go

@@ -606,6 +606,11 @@ func (s *StateDB) CommitTo(db hashtree.DatabaseWriter, blockNumber uint64, gc *h
 	return root, err
 }
 
+// HasDataCallback returns a GC callback function for a given state (identified by state root).
+// The callback tells if any state trie node, secure trie key preimage, contract code or contract storage
+// trie node is part of the given trie at the given position. The hash tree position encoding of state
+// trie nodes is identical to the general trie node position encoding. For contract code and storage
+// position encoding see contractCodePosition and storageTrieDb.
 func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(position, hash []byte) bool {
 	db := hashtree.NewReader(dbr, DbPrefix)
 	t, err := trie.New(root, db)
@@ -614,9 +619,13 @@ func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(positio
 	}
 	return func(position, hash []byte) bool {
 		lp := len(position)
-		if lp < 33 || (lp == 33 && position[32] < 5) {
+		if lp < 33 || (lp == 33 && position[32] < htContractCodeSuffix) {
+			// it should be a state trie node, check it there
 			return t.HasData(position, hash)
 		}
+		// it it either a code or a storage trie node, in either case we need the
+		// account entry to check. We do this manually with a "regular" (not secure)
+		// trie because we only know the address hash
 		addrHash := position[:32]
 		enc, err := t.TryGet(addrHash)
 		if len(enc) == 0 || err != nil {
@@ -627,13 +636,15 @@ func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(positio
 		if err := rlp.DecodeBytes(enc, &data); err != nil {
 			return false
 		}
-		if lp == 33 && position[32] == 5 {
+		if lp == 33 && position[32] == htContractCodeSuffix {
+			// if it is a code, the hash should match the currently present account's code hash
 			return bytes.Equal(hash, data.CodeHash)
 		}
-		if position[32] != 6 {
+		if position[32] != htContractStorageSuffix {
 			return false
 		}
 
+		// it is a storage trie node, check in the storage trie
 		st, err := trie.New(data.Root, &storageTrieDb{dbr: db, addrHash: addrHash})
 		if err != nil {
 			return false

trie/encoding.go

@@ -113,15 +113,24 @@ func hasTerm(s []byte) bool {
 	return len(s) > 0 && s[len(s)-1] == 16
 }
 
+const (
+	htEvenNibbleSuffix = 0
+	htOddNibbleSuffix  = 1
+	htSecTrieKeySuffix = 4
+)
+
+// hexToHashTreePos converts a hex encoded trie key prefix to a hash tree position.
+// Trie hash tree position encoding
+// - for even number of nibbles:   nibbles[0]*16+nibbles[1], ..., nibbles[i*2]*16+nibbles[i*2+1], 0
+// - for odd number of nibbles:    nibbles[0]*16+nibbles[1], ..., nibbles[i*2+1]*16+1
 func hexToHashTreePos(hex []byte) []byte {
-	terminator := byte(0)
+	terminator := byte(htEvenNibbleSuffix)
 	if hasTerm(hex) {
-		terminator = 2
 		hex = hex[:len(hex)-1]
 	}
 	buf := make([]byte, len(hex)/2+1)
 	if len(hex)&1 == 1 {
-		terminator += hex[len(hex)-1]<<4 + 1
+		terminator = hex[len(hex)-1]<<4 + htOddNibbleSuffix
 		hex = hex[:len(hex)-1]
 	}
 	decodeNibbles(hex, buf[:len(buf)-1])
@@ -129,18 +138,17 @@ func hexToHashTreePos(hex []byte) []byte {
 	return buf
 }
 
+// SecHashTreePos returns the hash tree position for secure trie key preimage entries
 func SecHashTreePos(hash []byte) []byte {
-	return append(hash, 4)
+	return append(hash, htSecTrieKeySuffix)
 }
 
+// hashTreePosToHex converts hash tree position (either of a trie node or a secure trie
+// key preimage entry) back to hex encoding
 func hashTreePosToHex(pos []byte) []byte {
 	base := keybytesToHex(pos)
 	base = base[:len(base)-1]
 	term := base[len(base)-1]
 	base = base[:len(base)-2+int(term&1)]
-	// apply terminator flag
-	if term >= 2 {
-		base = append(base, 16)
-	}
 	return base
 }

trie/trie.go

@@ -504,18 +504,28 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) {
 	return h.hash(t.root, db, nil, true)
 }
 
+// HasData checks whether a trie node or the entry belonging to a secure trie key
+// preimage is present in the trie
 func (t *Trie) HasData(position, hash []byte) bool {
+	secTrieKey := len(position) > 0 && position[len(position)-1] == htSecTrieKeySuffix
+	if secTrieKey && (len(position) != len(hash)+1 || !bytes.Equal(position[:len(hash)], hash)) {
+		// position for a secure trie key is always hash + htSecTrieKeySuffix
+		return false
+	}
 	hex := hashTreePosToHex(position)
 	//fmt.Println("pos", position, "hex", hex, "hash", hash)
 	n, err := t.ProveHexKey(hex, 0, nil)
 	if n == nil || err != nil {
 		return false
 	}
+	if secTrieKey {
+		// for secure trie keys we only care about whether the given trie contains an
+		// entry at that key, regardless of its contents
+		return true
+	}
 	hasher := newHasher(0, 0)
 	n, _, _ = hasher.hashChildren(n, nil, nil)
 	hn, _ := hasher.store(n, nil, nil, false)
 	nodeHash, ok := hn.(hashNode)
-	eq := ok && bytes.Equal(nodeHash, hash)
+	return ok && bytes.Equal(nodeHash, hash)
-	//fmt.Println("eq", eq, ok, nodeHash, hash)
-	return eq
 }