forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-17 01:13:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

core, trie: new state trie database format with garbage collection

Browse source
This commit is contained in:
Zsolt Felfoldi 2018-01-11 23:48:54 +01:00
parent 5395aa3009
commit b0c313407f
19 changed files with 616 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
core/blockchain.go
View file

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/hashtree"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
@ -105,14 +106,18 @@ type BlockChain struct {
quit chan struct{} // blockchain quit channel
running int32 // running must be called atomically
// procInterrupt must be atomically called
processing int32
procInterrupt int32 // interrupt signaler for block processing
wg sync.WaitGroup // chain processing wait group for shutting down
writeCounter uint64
engine consensus.Engine
processor Processor // block processor interface
validator Validator // block and state validator interface
vmConfig vm.Config
gc *hashtree.GarbageCollector
badBlocks *lru.Cache // Bad block cache
}
@ -126,6 +131,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
futureBlocks, _ := lru.New(maxFutureBlocks)
badBlocks, _ := lru.New(badBlockLimit)
//hashtree.Print(chainDb, []byte(state.DbPrefix))
bc := &BlockChain{
config: config,
chainDb: chainDb,
@ -167,11 +173,29 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co
}
}
}
bc.gc = hashtree.NewGarbageCollector(chainDb, []byte(state.DbPrefix), bc.hasDataCallback)
/*headBlock := bc.currentBlock.NumberU64()
if headBlock > 1000 {
bc.gc.FullGC(headBlock - 1000)
}*/
bc.gc.BackgroundGC(bc.CurrentBlock, &bc.processing, &bc.procInterrupt, &bc.wg)
// Take ownership of this particular state
go bc.update()
return bc, nil
}
func (bc *BlockChain) hasDataCallback(version uint64) func(position, hash []byte) bool {
header := bc.GetHeaderByNumber(version)
if header == nil {
return nil
}
return state.HasDataCallback(header.Root, bc.chainDb)
}
func (bc *BlockChain) getProcInterrupt() bool {
return atomic.LoadInt32(&bc.procInterrupt) == 1
}
@ -292,7 +316,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x…]", hash[:4])
}
if _, err := trie.NewSecure(block.Root(), bc.chainDb, 0); err != nil {
if _, err := trie.NewSecure(block.Root(), hashtree.NewReader(bc.chainDb, state.DbPrefix), 0); err != nil {
return err
}
// If all checks out, manually set the head block
@ -808,7 +832,7 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
if err := WriteBlock(batch, block); err != nil {
return NonStatTy, err
}
if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil {
if _, err := state.CommitTo(batch, block.NumberU64(), bc.gc, bc.config.IsEIP158(block.Number())); err != nil {
return NonStatTy, err
}
if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
@ -842,9 +866,13 @@ func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.R
} else {
status = SideStatTy
}
bc.gc.LockWrite()
if err := batch.Write(); err != nil {
bc.gc.UnlockWrite()
return NonStatTy, err
}
bc.gc.UnlockWrite()
// Set new head.
if status == CanonStatTy {
@ -888,6 +916,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
bc.chainmu.Lock()
defer bc.chainmu.Unlock()
atomic.StoreInt32(&bc.processing, 1)
defer atomic.StoreInt32(&bc.processing, 0)
// A queued approach to delivering events. This is generally
// faster than direct delivery and requires much less mutex
// acquiring.

2
core/chain_makers.go
View file

@ -192,7 +192,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
if b.engine != nil {
block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
// Write state changes to db
_, err := statedb.CommitTo(db, config.IsEIP158(b.header.Number))
_, err := statedb.CommitTo(db, b.header.Number.Uint64(), nil, config.IsEIP158(b.header.Number))
if err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}

17
core/database_util.go
View file

@ -25,12 +25,14 @@ import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// DatabaseReader wraps the Get method of a backing data store.
@ -58,8 +60,7 @@ var (
lookupPrefix = []byte("l") // lookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
preimagePrefix = "secure-key-" // preimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
configPrefix = []byte("ethereum-config-") // config prefix for the db
// Chain index prefixes (use `i` + single byte to avoid mixing data types).
BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
@ -532,20 +533,20 @@ func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) {
db.Delete(append(lookupPrefix, hash.Bytes()...))
}
// PreimageTable returns a Database instance with the key prefix for preimage entries.
func PreimageTable(db ethdb.Database) ethdb.Database {
return ethdb.NewTable(db, preimagePrefix)
func GetPreimage(db ethdb.Database, hash common.Hash) ([]byte, error) {
return db.Get(append([]byte(state.DbPrefix), trie.SecHashTreePos(hash.Bytes())...))
}
// WritePreimages writes the provided set of preimages to the database. `number` is the
// current block number, and is used for debug messages only.
func WritePreimages(db ethdb.Database, number uint64, preimages map[common.Hash][]byte) error {
table := PreimageTable(db)
table := ethdb.NewTable(db, state.DbPrefix)
batch := table.NewBatch()
hitCount := 0
for hash, preimage := range preimages {
if _, err := table.Get(hash.Bytes()); err != nil {
batch.Put(hash.Bytes(), preimage)
key := trie.SecHashTreePos(hash.Bytes())
if _, err := table.Get(key); err != nil {
batch.Put(key, preimage)
hitCount++
}
}

2
core/genesis.go
View file

@ -262,7 +262,7 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
if block.Number().Sign() != 0 {
return nil, fmt.Errorf("can't commit genesis block with number > 0")
}
if _, err := statedb.CommitTo(db, false); err != nil {
if _, err := statedb.CommitTo(db, 0, nil, false); err != nil {
return nil, fmt.Errorf("cannot write state: %v", err)
}
if err := WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty); err != nil {

232
core/hashtree/gc.go Normal file
View file

@ -0,0 +1,232 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package hashtree
import (
"bytes"
"encoding/binary"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/syndtr/goleveldb/leveldb/util"
)
func Print(db ethdb.Database, prefix []byte) {
it := db.(*ethdb.LDBDatabase).NewIterator()
defer it.Release()
cnt := 0
for it.Seek(prefix); it.Valid(); it.Next() {
key := it.Key()
if len(key) < len(prefix) || !bytes.Equal(key[:len(prefix)], prefix) {
return
}
value := it.Value()
cnt++
fmt.Printf("CNT %d KEY %x HASH %x VALUE %x\n", cnt, key[len(prefix):], crypto.Keccak256(value), value)
}
}
type hasDataFn func(version uint64) func(position, hash []byte) bool
type GarbageCollector struct {
db *ethdb.LDBDatabase
prefix []byte
hasData hasDataFn
gcBlock uint64
gcBlockHasData func(position, hash []byte) bool
delkeys [][]byte
keysChecked, keysRemoved uint64
refsChecked, refsRemoved uint64
writeCounter uint64
writeLock sync.Mutex
valid bool
}
func NewGarbageCollector(db ethdb.Database, prefix []byte, hasData hasDataFn) *GarbageCollector {
return &GarbageCollector{
db: db.(*ethdb.LDBDatabase),
prefix: prefix,
hasData: hasData,
}
}
func (g *GarbageCollector) run(startKey []byte, maxEntries uint64) (nextKey []byte) {
g.writeLock.Lock()
g.valid = true
g.writeLock.Unlock()
it := g.db.NewIterator()
g.delkeys = nil
defer func() {
it.Release()
g.writeLock.Lock()
if g.valid {
for _, key := range g.delkeys {
g.db.Delete(key)
}
g.keysRemoved += uint64(len(g.delkeys))
}
g.writeLock.Unlock()
var r util.Range
if nextKey == nil {
r = *util.BytesPrefix(g.prefix)
} else {
r.Limit = nextKey
}
r.Start = startKey
g.db.LDB().CompactRange(r)
}()
g.gcBlockHasData = g.hasData(g.gcBlock)
it.Seek(startKey)
for it.Valid() {
key := common.CopyBytes(it.Key())
//log.Info("key", "key", key)
if len(key) < len(g.prefix) || !bytes.Equal(key[:len(g.prefix)], g.prefix) {
return nil
}
if maxEntries == 0 {
nextKey = key
return nextKey
}
if len(key) >= len(g.prefix)+33 && key[len(key)-1] == 0 {
it.Next()
var refkeys [][]byte
for it.Valid() {
refkey := common.CopyBytes(it.Key())
//log.Info("ref", "key", refkey)
if len(refkey) >= len(key) && bytes.Equal(refkey[:len(key)-1], key[:len(key)-1]) {
if len(refkey) == len(key)+8 && refkey[len(key)+7] == 1 {
refkeys = append(refkeys, refkey)
} else {
log.Error("Invalid hashtree ref", "key", refkey)
}
it.Next()
} else {
break
}
}
g.gcEntry(key, refkeys)
maxEntries--
} else {
log.Error("Invalid hashtree entry", "key", key)
it.Next()
}
}
return nil
}
func (g *GarbageCollector) gcEntry(key []byte, refkeys [][]byte) {
refcount := len(refkeys)
keylen := len(key)
oldrefs := 0
for oldrefs < refcount {
version := binary.BigEndian.Uint64(refkeys[oldrefs][keylen-1 : keylen+7])
if version >= g.gcBlock {
break
}
oldrefs++
}
removerefs := 0
if oldrefs > 0 {
removerefs = oldrefs - 1
if oldrefs == refcount && !g.gcBlockHasData(key[len(g.prefix):keylen-33], key[keylen-33:keylen-1]) {
removerefs = refcount
}
}
g.keysChecked++
if removerefs == refcount {
g.delkeys = append(g.delkeys, key)
}
g.refsChecked += uint64(refcount)
g.refsRemoved += uint64(removerefs)
for i := 0; i < removerefs; i++ {
g.db.Delete(refkeys[i])
}
}
func (g *GarbageCollector) FullGC(block uint64) {
log.Info("Starting full GC", "block", block)
g.gcBlock = block
key := g.prefix
for key != nil {
key = g.run(key, 10000)
k := key
if len(k) > 8 {
k = k[:8]
}
log.Info("Running...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
}
log.Info("Finished full GC", "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
}
func (g *GarbageCollector) BackgroundGC(currentBlock func() *types.Block, processing, stop *int32, wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
var gcCounter uint64
key := g.prefix
for atomic.LoadInt32(stop) == 0 {
wc := atomic.LoadUint64(&g.writeCounter)
diff := wc - gcCounter
if diff > 10000 {
gcCounter = wc - 10000
diff = 10000
}
if diff >= 100 && atomic.LoadInt32(processing) == 0 {
gcCounter += 100
if key == nil {
key = g.prefix
}
headBlock := currentBlock().NumberU64()
if headBlock > 1000 {
g.gcBlock = headBlock - 1000
key = g.run(key, 1000)
k := key
if len(k) > 8 {
k = k[:8]
}
log.Info("Running GC...", "key", k, "keys checked", g.keysChecked, "keys removed", g.keysRemoved, "refs checked", g.refsChecked, "refs removed", g.refsRemoved)
}
} else {
time.Sleep(time.Second)
}
}
}()
}
func (g *GarbageCollector) LockWrite() {
g.writeLock.Lock()
g.valid = false
}
func (g *GarbageCollector) UnlockWrite() {
g.writeLock.Unlock()
}

107
core/hashtree/hashtree.go Normal file
View file

@ -0,0 +1,107 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package hashtree
import (
"encoding/binary"
"sync/atomic"
// "fmt"
)
type DatabaseReader interface {
Get([]byte) ([]byte, error)
Has([]byte) (bool, error)
}
type DatabaseWriter interface {
Put([]byte, []byte) error
}
type Reader struct {
db DatabaseReader
prefix []byte
lpf int
}
func NewReader(db DatabaseReader, prefix string) *Reader {
return &Reader{db, []byte(prefix), len(prefix)}
}
func (h *Reader) Get(position, hash []byte) ([]byte, error) {
lp, lh := len(position), len(hash)
key := make([]byte, h.lpf+lp+lh+1)
copy(key[:h.lpf], h.prefix)
copy(key[h.lpf:h.lpf+lp], position)
copy(key[h.lpf+lp:h.lpf+lp+lh], hash)
data, err := h.db.Get(key)
if err != nil {
//panic(nil)
//fmt.Printf("READ ERR %x %v\n", key, err)
}
return data, err
}
func (h *Reader) Has(position, hash []byte) (bool, error) {
lp, lh := len(position), len(hash)
key := make([]byte, h.lpf+lp+lh+1)
copy(key[:h.lpf], h.prefix)
copy(key[h.lpf:h.lpf+lp], position)
copy(key[h.lpf+lp:h.lpf+lp+lh], hash)
return h.db.Has(key)
}
func (h *Reader) Put(position, hash, data []byte) error {
panic(nil)
}
type Writer struct {
db DatabaseWriter
prefix []byte
lpf int
version uint64
versionEnc [8]byte
gc *GarbageCollector
}
func NewWriter(db DatabaseWriter, prefix string, version uint64, gc *GarbageCollector) *Writer {
w := &Writer{
db: db,
prefix: []byte(prefix),
lpf: len(prefix),
version: version,
gc: gc,
}
binary.BigEndian.PutUint64(w.versionEnc[:], version)
return w
}
func (w *Writer) Put(position, hash, data []byte) error {
if w.gc != nil {
atomic.AddUint64(&w.gc.writeCounter, 1)
}
lp, lh := len(position), len(hash)
key := make([]byte, w.lpf+lp+lh+9)
copy(key[:w.lpf], w.prefix)
copy(key[w.lpf:w.lpf+lp], position)
copy(key[w.lpf+lp:w.lpf+lp+lh], hash)
if err := w.db.Put(key[:w.lpf+lp+lh+1], data); err != nil {
return err
}
copy(key[w.lpf+lp+lh:w.lpf+lp+lh+8], w.versionEnc[:])
key[w.lpf+lp+lh+8] = 1
return w.db.Put(key, nil)
}

45
core/state/database.go
View file

@ -21,7 +21,7 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/core/hashtree"
"github.com/ethereum/go-ethereum/trie"
lru "github.com/hashicorp/golang-lru"
)
@ -29,6 +29,8 @@ import (
// Trie cache generation limit after which to evic trie nodes from memory.
var MaxTrieCacheGen = uint16(120)
var DbPrefix = "s"
const (
// Number of past tries to keep. This value is chosen such that
// reasonable chain reorg depths will hit an existing trie.
@ -65,13 +67,13 @@ type Trie interface {
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains cached trie nodes in memory.
func NewDatabase(db ethdb.Database) Database {
func NewDatabase(db hashtree.DatabaseReader) Database {
csc, _ := lru.New(codeSizeCacheSize)
return &cachingDB{db: db, codeSizeCache: csc}
return &cachingDB{db: hashtree.NewReader(db, DbPrefix), codeSizeCache: csc} //prefix
}
type cachingDB struct {
db ethdb.Database
db *hashtree.Reader
mu sync.Mutex
pastTries []*trie.SecureTrie
codeSizeCache *lru.Cache
@ -106,7 +108,7 @@ func (db *cachingDB) pushTrie(t *trie.SecureTrie) {
}
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
return trie.NewSecure(root, db.db, 0)
return trie.NewSecure(root, &storageTrieDb{dbr: db.db, addrHash: addrHash.Bytes()}, 0)
}
func (db *cachingDB) CopyTrie(t Trie) Trie {
@ -121,7 +123,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
}
func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
code, err := db.db.Get(codeHash[:])
code, err := db.db.Get(contractCodePosition(addrHash), codeHash[:])
if err == nil {
db.codeSizeCache.Add(codeHash, len(code))
}
@ -152,3 +154,34 @@ func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) {
}
return root, err
}
type storageTrieDb struct {
dbr trie.DatabaseReader
dbw trie.DatabaseWriter
addrHash []byte
}
func (s *storageTrieDb) position(position []byte) []byte {
pos := make([]byte, len(position)+33)
copy(pos[:32], s.addrHash)
pos[32] = 6
copy(pos[33:], position)
return pos
}
func (s *storageTrieDb) Put(position, hash, data []byte) error {
return s.dbw.Put(s.position(position), hash, data)
}
func (s *storageTrieDb) Get(position, hash []byte) ([]byte, error) {
//fmt.Printf("GetStorage %x %x %x\n", s.addrHash, position, hash)
return s.dbr.Get(s.position(position), hash)
}
func (s *storageTrieDb) Has(position, hash []byte) (bool, error) {
return s.dbr.Has(s.position(position), hash)
}
func contractCodePosition(addrHash common.Hash) []byte {
return append(addrHash.Bytes(), 5)
}

5
core/state/state_object.go
View file

@ -113,10 +113,11 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a
if data.CodeHash == nil {
data.CodeHash = emptyCodeHash
}
addrHash := crypto.Keccak256Hash(address[:])
return &stateObject{
db: db,
address: address,
addrHash: crypto.Keccak256Hash(address[:]),
addrHash: addrHash,
data: data,
cachedStorage: make(Storage),
dirtyStorage: make(Storage),
@ -243,7 +244,7 @@ func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error
if self.dbErr != nil {
return self.dbErr
}
root, err := self.trie.CommitTo(dbw)
root, err := self.trie.CommitTo(&storageTrieDb{dbw: dbw, addrHash: self.addrHash.Bytes()})
if err == nil {
self.data.Root = root
}

43
core/state/statedb.go
View file

@ -18,12 +18,14 @@
package state
import (
"bytes"
"fmt"
"math/big"
"sort"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/hashtree"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
@ -569,9 +571,10 @@ func (s *StateDB) clearJournalAndRefund() {
}
// CommitTo writes the state to the given database.
func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (root common.Hash, err error) {
func (s *StateDB) CommitTo(db hashtree.DatabaseWriter, blockNumber uint64, gc *hashtree.GarbageCollector, deleteEmptyObjects bool) (root common.Hash, err error) {
defer s.clearJournalAndRefund()
dbw := hashtree.NewWriter(db, DbPrefix, blockNumber, gc)
// Commit objects to the trie.
for addr, stateObject := range s.stateObjects {
_, isDirty := s.stateObjectsDirty[addr]
@ -583,7 +586,7 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
case isDirty:
// Write any contract code associated with the state object
if stateObject.code != nil && stateObject.dirtyCode {
if err := dbw.Put(stateObject.CodeHash(), stateObject.code); err != nil {
if err := dbw.Put(contractCodePosition(stateObject.addrHash), stateObject.CodeHash(), stateObject.code); err != nil {
return common.Hash{}, err
}
stateObject.dirtyCode = false
@ -602,3 +605,39 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro
log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads())
return root, err
}
func HasDataCallback(root common.Hash, dbr hashtree.DatabaseReader) func(position, hash []byte) bool {
db := hashtree.NewReader(dbr, DbPrefix)
t, err := trie.New(root, db)
if err != nil {
panic(err)
}
return func(position, hash []byte) bool {
lp := len(position)
if lp < 33 || (lp == 33 && position[32] < 5) {
return t.HasData(position, hash)
}
addrHash := position[:32]
enc, err := t.TryGet(addrHash)
if len(enc) == 0 || err != nil {
return false
}
var data Account
if err := rlp.DecodeBytes(enc, &data); err != nil {
return false
}
if lp == 33 && position[32] == 5 {
return bytes.Equal(hash, data.CodeHash)
}
if position[32] != 6 {
return false
}
st, err := trie.New(data.Root, &storageTrieDb{dbr: db, addrHash: addrHash})
if err != nil {
return false
}
return st.HasData(position[33:], hash)
}
}

8
eth/api.go
View file

@ -28,6 +28,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/hashtree"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
@ -343,8 +344,7 @@ func NewPrivateDebugAPI(config *params.ChainConfig, eth *Ethereum) *PrivateDebug
// Preimage is a debug API function that returns the preimage for a sha3 hash, if known.
func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) {
db := core.PreimageTable(api.eth.ChainDb())
return db.Get(hash.Bytes())
return core.GetPreimage(api.eth.ChainDb(), hash)
}
// GetBadBLocks returns a list of the last 'bad blocks' that the client has seen on the network
@ -462,11 +462,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
oldTrie, err := trie.NewSecure(startBlock.Root(), api.eth.chainDb, 0)
oldTrie, err := trie.NewSecure(startBlock.Root(), hashtree.NewReader(api.eth.chainDb, state.DbPrefix), 0)
if err != nil {
return nil, err
}
newTrie, err := trie.NewSecure(endBlock.Root(), api.eth.chainDb, 0)
newTrie, err := trie.NewSecure(endBlock.Root(), hashtree.NewReader(api.eth.chainDb, state.DbPrefix), 0)
if err != nil {
return nil, err
}

36
eth/api_tracer.go
View file

@ -123,24 +123,24 @@ func (db *ephemeralDatabase) Get(key []byte) ([]byte, error) {
// state.
func (db *ephemeralDatabase) Prune(root common.Hash) {
// Pull the still relevant state data into memory
sync := state.NewStateSync(root, db.diskdb)
for sync.Pending() > 0 {
hash := sync.Missing(1)[0]
/* sync := state.NewStateSync(root, db.diskdb)
for sync.Pending() > 0 {
hash := sync.Missing(1)[0]
// Move the next trie node from the memory layer into a sync struct
node, err := db.memdb.Get(hash[:])
if err != nil {
panic(err) // memdb must have the data
// Move the next trie node from the memory layer into a sync struct
node, err := db.memdb.Get(hash[:])
if err != nil {
panic(err) // memdb must have the data
}
if _, _, err := sync.Process([]trie.SyncResult{{Hash: hash, Data: node}}); err != nil {
panic(err) // it's not possible to fail processing a node
}
}
if _, _, err := sync.Process([]trie.SyncResult{{Hash: hash, Data: node}}); err != nil {
panic(err) // it's not possible to fail processing a node
}
}
// Discard the old memory layer and write a new one
db.memdb, _ = ethdb.NewMemDatabaseWithCap(db.memdb.Len())
if _, err := sync.Commit(db); err != nil {
panic(err) // writing into a memdb cannot fail
}
// Discard the old memory layer and write a new one
db.memdb, _ = ethdb.NewMemDatabaseWithCap(db.memdb.Len())
if _, err := sync.Commit(db); err != nil {
panic(err) // writing into a memdb cannot fail
}*/
}
// TraceChain returns the structured logs created during the execution of EVM
@ -340,7 +340,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
break
}
// Finalize the state so any modifications are written to the trie
root, err := statedb.CommitTo(db, true)
root, err := statedb.CommitTo(db, number, nil, true)
if err != nil {
failed = err
break
@ -587,7 +587,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
return nil, err
}
// Finalize the state so any modifications are written to the trie
root, err := statedb.CommitTo(db, true)
root, err := statedb.CommitTo(db, block.NumberU64(), nil, true)
if err != nil {
return nil, err
}

4
ethdb/database.go
View file

@ -347,6 +347,10 @@ func (dt *table) NewBatch() Batch {
return &tableBatch{dt.db.NewBatch(), dt.prefix}
}
func NewBatchTable(batch Batch, prefix string) Batch {
return &tableBatch{batch, prefix}
}
func (tb *tableBatch) Put(key, value []byte) error {
return tb.batch.Put(append([]byte(tb.prefix), key...), value)
}

4
tests/state_test_util.go
View file

@ -147,7 +147,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD
if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) {
return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs)
}
root, _ := statedb.CommitTo(db, config.IsEIP158(block.Number()))
root, _ := statedb.CommitTo(db, block.NumberU64(), nil, config.IsEIP158(block.Number()))
if root != common.Hash(post.Root) {
return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root)
}
@ -170,7 +170,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB
}
}
// Commit and re-open to start with a clean state.
root, _ := statedb.CommitTo(db, false)
root, _ := statedb.CommitTo(db, 0, nil, false)
statedb, _ = state.New(root, sdb)
return statedb
}

32
trie/encoding.go
View file

@ -112,3 +112,35 @@ func prefixLen(a, b []byte) int {
func hasTerm(s []byte) bool {
return len(s) > 0 && s[len(s)-1] == 16
}
func hexToHashTreePos(hex []byte) []byte {
terminator := byte(0)
if hasTerm(hex) {
terminator = 2
hex = hex[:len(hex)-1]
}
buf := make([]byte, len(hex)/2+1)
if len(hex)&1 == 1 {
terminator += hex[len(hex)-1]<<4 + 1
hex = hex[:len(hex)-1]
}
decodeNibbles(hex, buf[:len(buf)-1])
buf[len(buf)-1] = terminator
return buf
}
func SecHashTreePos(hash []byte) []byte {
return append(hash, 4)
}
func hashTreePosToHex(pos []byte) []byte {
base := keybytesToHex(pos)
base = base[:len(base)-1]
term := base[len(base)-1]
base = base[:len(base)-2+int(term&1)]
// apply terminator flag
if term >= 2 {
base = append(base, 16)
}
return base
}

20
trie/hasher.go
View file

@ -70,7 +70,7 @@ func (h *hasher) returnCalculator(calculator *calculator) {
// hash collapses a node down into a hash node, also returning a copy of the
// original node initialized with the computed hash to replace the original one.
func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) {
func (h *hasher) hash(n node, db DatabaseWriter, prefix []byte, force bool) (node, node, error) {
// If we're not storing the node, just hashing, use available cached data
if hash, dirty := n.cache(); hash != nil {
if db == nil {
@ -87,11 +87,11 @@ func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error)
}
}
// Trie not processed yet or needs storage, walk the children
collapsed, cached, err := h.hashChildren(n, db)
collapsed, cached, err := h.hashChildren(n, db, prefix)
if err != nil {
return hashNode{}, n, err
}
hashed, err := h.store(collapsed, db, force)
hashed, err := h.store(collapsed, db, prefix, force)
if err != nil {
return hashNode{}, n, err
}
@ -117,7 +117,7 @@ func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error)
// hashChildren replaces the children of a node with their hashes if the encoded
// size of the child is larger than a hash, returning the collapsed node as well
// as a replacement for the original node with the child hashes cached in.
func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, error) {
func (h *hasher) hashChildren(original node, db DatabaseWriter, prefix []byte) (node, node, error) {
var err error
switch n := original.(type) {
@ -128,7 +128,7 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err
cached.Key = common.CopyBytes(n.Key)
if _, ok := n.Val.(valueNode); !ok {
collapsed.Val, cached.Val, err = h.hash(n.Val, db, false)
collapsed.Val, cached.Val, err = h.hash(n.Val, db, append(prefix, n.Key...), false)
if err != nil {
return original, original, err
}
@ -155,7 +155,11 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err
}
// Hash all other children properly
var herr error
collapsed.Children[index], cached.Children[index], herr = h.hash(n.Children[index], db, false)
// copy prefix in order to be thread safe
childPrefix := make([]byte, len(prefix)+1)
copy(childPrefix[:len(prefix)], prefix)
childPrefix[len(prefix)] = byte(index)
collapsed.Children[index], cached.Children[index], herr = h.hash(n.Children[index], db, childPrefix, false)
if herr != nil {
h.mu.Lock() // rarely if ever locked, no congenstion
err = herr
@ -197,7 +201,7 @@ func (h *hasher) hashChildren(original node, db DatabaseWriter) (node, node, err
}
}
func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
func (h *hasher) store(n node, db DatabaseWriter, prefix []byte, force bool) (node, error) {
// Don't store hashes or empty nodes.
if _, isHash := n.(hashNode); n == nil || isHash {
return n, nil
@ -221,7 +225,7 @@ func (h *hasher) store(n node, db DatabaseWriter, force bool) (node, error) {
if db != nil {
// db might be a leveldb batch, which is not safe for concurrent writes
h.mu.Lock()
err := db.Put(hash, calculator.buffer.Bytes())
err := db.Put(hexToHashTreePos(prefix), hash, calculator.buffer.Bytes())
h.mu.Unlock()
return hash, err

70
trie/proof.go
View file

@ -26,6 +26,14 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
type ProofReader interface {
Get(hash []byte) (value []byte, err error)
}
type ProofWriter interface {
Put(hash, data []byte) error
}
// Prove constructs a merkle proof for key. The result contains all
// encoded nodes on the path to the value at key. The value itself is
// also included in the last node and can be retrieved by verifying
@ -35,65 +43,73 @@ import (
// contains all nodes of the longest existing prefix of the key
// (at least the root node), ending with the node that proves the
// absence of the key.
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb DatabaseWriter) error {
// Collect all nodes on the path to key.
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ProofWriter) error {
key = keybytesToHex(key)
_, err := t.ProveHexKey(key, fromLevel, proofDb)
return err
}
func (t *Trie) ProveHexKey(key []byte, fromLevel uint, proofDb ProofWriter) (node, error) {
// Collect all nodes on the path to key.
nodes := []node{}
tn := t.root
for len(key) > 0 && tn != nil {
ptr := 0
for len(key) > ptr && tn != nil {
switch n := tn.(type) {
case *shortNode:
if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) {
if len(key) < ptr+len(n.Key) || !bytes.Equal(n.Key, key[ptr:ptr+len(n.Key)]) {
// The trie doesn't contain the key.
tn = nil
} else {
tn = n.Val
key = key[len(n.Key):]
ptr += len(n.Key)
}
nodes = append(nodes, n)
case *fullNode:
tn = n.Children[key[0]]
key = key[1:]
tn = n.Children[key[ptr]]
ptr++
nodes = append(nodes, n)
case hashNode:
var err error
tn, err = t.resolveHash(n, nil)
tn, err = t.resolveHash(n, key[:ptr])
if err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
return err
return nil, err
}
default:
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
}
}
hasher := newHasher(0, 0)
for i, n := range nodes {
// Don't bother checking for errors here since hasher panics
// if encoding doesn't work and we're not writing to any database.
n, _, _ = hasher.hashChildren(n, nil)
hn, _ := hasher.store(n, nil, false)
if hash, ok := hn.(hashNode); ok || i == 0 {
// If the node's database encoding is a hash (or is the
// root node), it becomes a proof element.
if fromLevel > 0 {
fromLevel--
} else {
enc, _ := rlp.EncodeToBytes(n)
if !ok {
hash = crypto.Keccak256(enc)
if proofDb != nil {
hasher := newHasher(0, 0)
for i, n := range nodes {
// Don't bother checking for errors here since hasher panics
// if encoding doesn't work and we're not writing to any database.
n, _, _ = hasher.hashChildren(n, nil, nil)
hn, _ := hasher.store(n, nil, nil, false)
if hash, ok := hn.(hashNode); ok || i == 0 {
// If the node's database encoding is a hash (or is the
// root node), it becomes a proof element.
if fromLevel > 0 {
fromLevel--
} else {
enc, _ := rlp.EncodeToBytes(n)
if !ok {
hash = crypto.Keccak256(enc)
}
proofDb.Put(hash, enc)
}
proofDb.Put(hash, enc)
}
}
}
return nil
return tn, nil
}
// VerifyProof checks merkle proofs. The given proof must contain the
// value for key in a trie with the given root hash. VerifyProof
// returns an error if the proof contains invalid trie nodes or the
// wrong value.
func VerifyProof(rootHash common.Hash, key []byte, proofDb DatabaseReader) (value []byte, err error, nodes int) {
func VerifyProof(rootHash common.Hash, key []byte, proofDb ProofReader) (value []byte, err error, nodes int) {
key = keybytesToHex(key)
wantHash := rootHash[:]
for i := 0; ; i++ {

18
trie/secure_trie.go
View file

@ -23,9 +23,7 @@ import (
"github.com/ethereum/go-ethereum/log"
)
var secureKeyPrefix = []byte("secure-key-")
const secureKeyLength = 11 + 32 // Length of the above prefix + 32byte hash
const secureKeyLength = 100 //???
// SecureTrie wraps a trie with key hashing. In a secure trie, all
// access operations hash the key using keccak256. This prevents
@ -135,7 +133,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
key, _ := t.trie.db.Get(t.secKey(shaKey))
key, _ := t.trie.db.Get(SecHashTreePos(shaKey), shaKey)
return key
}
@ -176,7 +174,8 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator {
func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
if len(t.getSecKeyCache()) > 0 {
for hk, key := range t.secKeyCache {
if err := db.Put(t.secKey([]byte(hk)), key); err != nil {
shaKey := []byte(hk)
if err := db.Put(SecHashTreePos(shaKey), shaKey, key); err != nil {
return common.Hash{}, err
}
}
@ -185,15 +184,6 @@ func (t *SecureTrie) CommitTo(db DatabaseWriter) (root common.Hash, err error) {
return t.trie.CommitTo(db)
}
// secKey returns the database key for the preimage of key, as an ephemeral buffer.
// The caller must not hold onto the return value because it will become
// invalid on the next call to hashKey or secKey.
func (t *SecureTrie) secKey(key []byte) []byte {
buf := append(t.secKeyBuf[:0], secureKeyPrefix...)
buf = append(buf, key...)
return buf
}
// hashKey returns the hash of key as an ephemeral buffer.
// The caller must not hold onto the return value because it will become
// invalid on the next call to hashKey or secKey.

13
trie/sync.go
View file

@ -41,6 +41,7 @@ type request struct {
parents []*request // Parent state nodes referencing this entry (notify all upon completion)
depth int // Depth level within the trie the node is located to prioritise DFS
deps int // Number of dependencies before allowed to commit this node
prefix []byte // key prefix leading to the trie node
callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
}
@ -104,7 +105,8 @@ func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, c
return
}
key := root.Bytes()
blob, _ := s.database.Get(key)
panic(nil) // add position
blob, _ := s.database.Get(nil, key)
if local, err := decodeNode(key, blob, 0); local != nil && err == nil {
return
}
@ -138,7 +140,8 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash)
if _, ok := s.membatch.batch[hash]; ok {
return
}
if ok, _ := s.database.Has(hash.Bytes()); ok {
panic(nil) // add position
if ok, _ := s.database.Has(nil, hash.Bytes()); ok {
return
}
// Assemble the new sub-trie sync request
@ -220,7 +223,8 @@ func (s *TrieSync) Process(results []SyncResult) (bool, int, error) {
func (s *TrieSync) Commit(dbw DatabaseWriter) (int, error) {
// Dump the membatch into a database dbw
for i, key := range s.membatch.order {
if err := dbw.Put(key[:], s.membatch.batch[key]); err != nil {
panic(nil) // add position
if err := dbw.Put(nil, key[:], s.membatch.batch[key]); err != nil {
return i, err
}
}
@ -296,7 +300,8 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) {
if _, ok := s.membatch.batch[hash]; ok {
continue
}
if ok, _ := s.database.Has(node); ok {
panic(nil) // add position
if ok, _ := s.database.Has(req.prefix, node); ok {
continue
}
// Locally unknown node, schedule for retrieval

28
trie/trie.go
View file

@ -65,8 +65,8 @@ type Database interface {
// DatabaseReader wraps the Get method of a backing store for the trie.
type DatabaseReader interface {
Get(key []byte) (value []byte, err error)
Has(key []byte) (bool, error)
Get(position, hash []byte) (value []byte, err error)
Has(position, hash []byte) (bool, error)
}
// DatabaseWriter wraps the Put method of a backing store for the trie.
@ -74,7 +74,7 @@ type DatabaseWriter interface {
// Put stores the mapping key->value in the database.
// Implementations must not hold onto the value bytes, the trie
// will reuse the slice across calls to Put.
Put(key, value []byte) error
Put(position, hash, data []byte) error
}
// Trie is a Merkle Patricia Trie.
@ -389,7 +389,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
// shortNode{..., shortNode{...}}. Since the entry
// might not be loaded yet, resolve it just for this
// check.
cnode, err := t.resolve(n.Children[pos], prefix)
cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos)))
if err != nil {
return false, nil, err
}
@ -447,7 +447,7 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
cacheMissCounter.Inc(1)
enc, err := t.db.Get(n)
enc, err := t.db.Get(hexToHashTreePos(prefix), n)
if err != nil || enc == nil {
return nil, &MissingNodeError{NodeHash: common.BytesToHash(n), Path: prefix}
}
@ -501,5 +501,21 @@ func (t *Trie) hashRoot(db DatabaseWriter) (node, node, error) {
return hashNode(emptyRoot.Bytes()), nil, nil
}
h := newHasher(t.cachegen, t.cachelimit)
return h.hash(t.root, db, true)
return h.hash(t.root, db, nil, true)
}
func (t *Trie) HasData(position, hash []byte) bool {
hex := hashTreePosToHex(position)
//fmt.Println("pos", position, "hex", hex, "hash", hash)
n, err := t.ProveHexKey(hex, 0, nil)
if n == nil || err != nil {
return false
}
hasher := newHasher(0, 0)
n, _, _ = hasher.hashChildren(n, nil, nil)
hn, _ := hasher.store(n, nil, nil, false)
nodeHash, ok := hn.(hashNode)
eq := ok && bytes.Equal(nodeHash, hash)
//fmt.Println("eq", eq, ok, nodeHash, hash)
return eq
}