forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 22:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

dev: fix: more wsl lint issues

Browse source
This commit is contained in:
marcello33 2023-06-15 15:10:57 +02:00
parent 131d7b221f
commit 6223765aab
37 changed files with 407 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
core/state/database.go
View file

@ -195,7 +195,9 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error
if len(code) > 0 {
return code, nil
}
code = rawdb.ReadCode(db.disk, codeHash)
if len(code) > 0 {
db.codeCache.Add(codeHash, code)
db.codeSizeCache.Add(codeHash, len(code))
@ -212,7 +214,9 @@ func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]b
if len(code) > 0 {
return code, nil
}
code = rawdb.ReadCodeWithPrefix(db.disk, codeHash)
if len(code) > 0 {
db.codeCache.Add(codeHash, code)
db.codeSizeCache.Add(codeHash, len(code))

2
core/state/dump.go
View file

@ -169,10 +169,12 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
if !conf.SkipStorage {
account.Storage = make(map[common.Hash]string)
tr, err := obj.getTrie(s.db)
if err != nil {
log.Error("Failed to load storage trie", "err", err)
continue
}
storageIt := trie.NewIterator(tr.NodeIterator(nil))
for storageIt.Next() {
_, content, _, err := rlp.Split(storageIt.Value)

3
core/state/iterator.go
View file

@ -109,7 +109,9 @@ func (it *NodeIterator) step() error {
if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil {
return err
}
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root)
if err != nil {
return err
}
@ -117,6 +119,7 @@ func (it *NodeIterator) step() error {
if !it.dataIt.Next(true) {
it.dataIt = nil
}
if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) {
it.codeHash = common.BytesToHash(account.CodeHash)
addrHash := common.BytesToHash(it.stateIt.LeafKey())

1
core/state/journal.go
View file

@ -159,6 +159,7 @@ func (ch createObjectChange) dirtied() *common.Address {
func (ch resetObjectChange) revert(s *StateDB) {
s.setStateObject(ch.prev)
RevertWrite(s, blockstm.NewAddressKey(ch.prev.address))
if !ch.prevdestruct {
delete(s.stateObjectsDestruct, ch.prev.address)
}

6
core/state/pruner/pruner.go
View file

@ -86,6 +86,7 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
if headBlock == nil {
return nil, errors.New("failed to load head block")
}
snapconfig := snapshot.Config{
CacheSize: 256,
Recovery: false,
@ -101,6 +102,7 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
log.Warn("Sanitizing bloomfilter size", "provided(MB)", config.BloomSize, "updated(MB)", 256)
config.BloomSize = 256
}
stateBloom, err := newStateBloomWithSize(config.BloomSize)
if err != nil {
return nil, err
@ -327,6 +329,7 @@ func (p *Pruner) Prune(root common.Hash) error {
if err := extractGenesis(p.db, p.stateBloom); err != nil {
return err
}
filterName := bloomFilterName(p.config.Datadir, root)
log.Info("Writing state bloom to disk", "name", filterName)
@ -418,6 +421,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil {
return errors.New("missing genesis block")
}
t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
if err != nil {
return err
@ -437,6 +441,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil {
return err
}
if acc.Root != types.EmptyRootHash {
id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
@ -454,6 +459,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
return storageIter.Error()
}
}
if !bytes.Equal(acc.CodeHash, types.EmptyCodeHash.Bytes()) {
stateBloom.Put(acc.CodeHash, nil)
}

1
core/state/snapshot/account.go
View file

@ -45,6 +45,7 @@ func SlimAccount(nonce uint64, balance *big.Int, root common.Hash, codehash []by
if root != types.EmptyRootHash {
slim.Root = root[:]
}
if !bytes.Equal(codehash, types.EmptyCodeHash[:]) {
slim.CodeHash = codehash
}

23
core/state/snapshot/context.go
View file

@ -82,6 +82,7 @@ func (gs *generatorStats) Log(msg string, root common.Hash, marker []byte) {
}...)
}
}
log.Info(msg, ctx...)
}
@ -105,6 +106,7 @@ func newGeneratorContext(stats *generatorStats, db ethdb.KeyValueStore, accMarke
}
ctx.openIterator(snapAccount, accMarker)
ctx.openIterator(snapStorage, storageMarker)
return ctx
}
@ -115,8 +117,10 @@ func (ctx *generatorContext) openIterator(kind string, start []byte) {
if kind == snapAccount {
iter := ctx.db.NewIterator(rawdb.SnapshotAccountPrefix, start)
ctx.account = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+common.HashLength))
return
}
iter := ctx.db.NewIterator(rawdb.SnapshotStoragePrefix, start)
ctx.storage = newHoldableIterator(rawdb.NewKeyLengthIterator(iter, 1+2*common.HashLength))
}
@ -130,17 +134,23 @@ func (ctx *generatorContext) reopenIterator(kind string) {
if kind == snapStorage {
iter = ctx.storage
}
hasNext := iter.Next()
if !hasNext {
// Iterator exhausted, release forever and create an already exhausted virtual iterator
iter.Release()
if kind == snapAccount {
ctx.account = newHoldableIterator(memorydb.New().NewIterator(nil, nil))
return
}
ctx.storage = newHoldableIterator(memorydb.New().NewIterator(nil, nil))
return
}
next := iter.Key()
iter.Release()
ctx.openIterator(kind, next[1:])
@ -157,6 +167,7 @@ func (ctx *generatorContext) iterator(kind string) *holdableIterator {
if kind == snapAccount {
return ctx.account
}
return ctx.storage
}
@ -170,6 +181,7 @@ func (ctx *generatorContext) removeStorageBefore(account common.Hash) {
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
key := iter.Key()
if bytes.Compare(key[1:1+common.HashLength], account.Bytes()) >= 0 {
@ -178,12 +190,15 @@ func (ctx *generatorContext) removeStorageBefore(account common.Hash) {
}
count++
ctx.batch.Delete(key)
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
}
}
ctx.stats.dangling += count
snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds())
}
@ -197,18 +212,22 @@ func (ctx *generatorContext) removeStorageAt(account common.Hash) error {
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
key := iter.Key()
cmp := bytes.Compare(key[1:1+common.HashLength], account.Bytes())
if cmp < 0 {
return errors.New("invalid iterator position")
}
if cmp > 0 {
iter.Hold()
break
}
count++
ctx.batch.Delete(key)
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
@ -216,6 +235,7 @@ func (ctx *generatorContext) removeStorageAt(account common.Hash) error {
}
snapWipedStorageMeter.Mark(count)
snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds())
return nil
}
@ -227,14 +247,17 @@ func (ctx *generatorContext) removeStorageLeft() {
start = time.Now()
iter = ctx.storage
)
for iter.Next() {
count++
ctx.batch.Delete(iter.Key())
if ctx.batch.ValueSize() > ethdb.IdealBatchSize {
ctx.batch.Write()
ctx.batch.Reset()
}
}
ctx.stats.dangling += count
snapDanglingStorageMeter.Mark(int64(count))
snapStorageCleanCounter.Inc(time.Since(start).Nanoseconds())

1
core/state/snapshot/conversion.go
View file

@ -369,6 +369,7 @@ func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash
rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme)
}
}
t := trie.NewStackTrieWithOwner(nodeWriter, owner)
for leaf := range in {
t.Update(leaf.key[:], leaf.value)

27
core/state/snapshot/generate.go
View file

@ -361,11 +361,14 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
root, nodes := snapTrie.Commit(false)
if nodes != nil {
tdb.Update(trie.NewWithNodeSet(nodes))
tdb.Commit(root, false)
}
resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte {
return rawdb.ReadTrieNode(mdb, owner, path, hash, tdb.Scheme())
}
@ -397,6 +400,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
start = time.Now()
internal time.Duration
)
nodeIt.AddResolver(resolver)
for iter.Next() {
@ -457,6 +461,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
} else {
snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
}
logger.Debug("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last),
"count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted)
@ -473,6 +478,7 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error
case abort = <-dl.genAbort:
default:
}
if ctx.batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
if bytes.Compare(current, dl.genMarker) < 0 {
log.Error("Snapshot generator went backwards", "current", fmt.Sprintf("%x", current), "genMarker", fmt.Sprintf("%x", dl.genMarker))
@ -485,6 +491,7 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error
if err := ctx.batch.Write(); err != nil {
return err
}
ctx.batch.Reset()
dl.lock.Lock()
@ -499,10 +506,12 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error
ctx.reopenIterator(snapAccount)
ctx.reopenIterator(snapStorage)
}
if time.Since(ctx.logged) > 8*time.Second {
ctx.stats.Log("Generating state snapshot", dl.root, current)
ctx.logged = time.Now()
}
return nil
}
@ -518,14 +527,17 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Has
if delete {
rawdb.DeleteStorageSnapshot(ctx.batch, account, common.BytesToHash(key))
snapWipedStorageMeter.Mark(1)
return nil
}
if write {
rawdb.WriteStorageSnapshot(ctx.batch, account, common.BytesToHash(key), val)
snapGeneratedStorageMeter.Mark(1)
} else {
snapRecoveredStorageMeter.Mark(1)
}
ctx.stats.storage += common.StorageSize(1 + 2*common.HashLength + len(val))
ctx.stats.slots++
@ -537,9 +549,11 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Has
}
// Loop for re-generating the missing storage slots.
var origin = common.CopyBytes(storeMarker)
for {
id := trie.StorageTrieID(stateRoot, account, storageRoot)
exhausted, last, err := dl.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil)
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
@ -547,10 +561,12 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Has
if exhausted {
break
}
if origin = increaseKey(last); origin == nil {
break // special case, the last is 0xffffffff...fff
}
}
return nil
}
@ -589,6 +605,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if bytes.Equal(acc.CodeHash, types.EmptyCodeHash[:]) {
dataLen -= 32
}
if acc.Root == types.EmptyRootHash {
dataLen -= 32
}
@ -599,6 +616,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
rawdb.WriteAccountSnapshot(ctx.batch, account, data)
snapGeneratedAccountMeter.Mark(1)
}
ctx.stats.storage += common.StorageSize(1 + common.HashLength + dataLen)
ctx.stats.accounts++
}
@ -612,6 +630,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if err := dl.checkAndFlush(ctx, marker); err != nil {
return err
}
snapAccountWriteCounter.Inc(time.Since(start).Nanoseconds()) // let's count flush time as well
// If the iterated account is the contract, create a further loop to
@ -637,6 +656,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if len(accMarker) > 0 {
accountRange = 1
}
origin := common.CopyBytes(accMarker)
for {
id := trie.StateTrieID(dl.root)
@ -644,6 +664,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if err != nil {
return err // The procedure it aborted, either by external signal or internal error.
}
origin = increaseKey(last)
// Last step, cleanup the storages after the last account.
@ -654,6 +675,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
}
accountRange = accountCheckRange
}
return nil
}
@ -666,9 +688,11 @@ func (dl *diskLayer) generate(stats *generatorStats) {
accMarker []byte
abort chan *generatorStats
)
if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
accMarker = dl.genMarker[:common.HashLength]
}
stats.Log("Resuming state snapshot generation", dl.root, dl.genMarker)
// Initialize the global generator context. The snapshot iterators are
@ -692,12 +716,14 @@ func (dl *diskLayer) generate(stats *generatorStats) {
abort = <-dl.genAbort
}
abort <- stats
return
}
// Snapshot fully generated, set the marker to nil.
// Note even there is nothing to commit, persist the
// generator anyway to mark the snapshot is complete.
journalProgress(ctx.batch, nil, stats)
if err := ctx.batch.Write(); err != nil {
log.Error("Failed to flush batch", "err", err)
@ -705,6 +731,7 @@ func (dl *diskLayer) generate(stats *generatorStats) {
abort <- stats
return
}
ctx.batch.Reset()
log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots,

20
core/state/snapshot/generate_test.go
View file

@ -35,10 +35,13 @@ import (
func hashData(input []byte) common.Hash {
var hasher = sha3.NewLegacyKeccak256()
var hash common.Hash
hasher.Reset()
hasher.Write(input)
hasher.Sum(hash[:0])
return hash
}
@ -135,6 +138,7 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
if snapRoot != trieRoot {
t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot)
}
if err := CheckDanglingStorage(snap.diskdb); err != nil {
t.Fatalf("Detected dangling storages: %v", err)
}
@ -188,10 +192,13 @@ func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string
for i, k := range keys {
stTrie.MustUpdate([]byte(k), []byte(vals[i]))
}
if !commit {
return stTrie.Hash().Bytes()
}
root, nodes := stTrie.Commit(false)
if nodes != nil {
t.nodes.Merge(nodes)
}
@ -205,6 +212,7 @@ func (t *testHelper) Commit() common.Hash {
}
t.triedb.Update(t.nodes)
t.triedb.Commit(root, false)
return root
}
@ -517,12 +525,14 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
}
root := helper.Commit()
// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
t.Fatalf("expected snap storage to exist")
}
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
@ -552,6 +562,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
if false {
enableLogging()
}
helper := newHelper()
{
// Account one in the trie
@ -580,6 +591,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
}
}
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
@ -609,6 +621,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
if false {
enableLogging()
}
helper := newHelper()
{
acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}
@ -624,6 +637,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val)
}
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
@ -646,6 +660,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
if false {
enableLogging()
}
helper := newHelper()
{
acc := &Account{Balance: big.NewInt(1), Root: types.EmptyRootHash.Bytes(), CodeHash: types.EmptyCodeHash.Bytes()}
@ -659,6 +674,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk)
rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk)
}
root, snap := helper.CommitAndGenerate()
select {
case <-snap.genPending:
@ -689,6 +705,7 @@ func TestGenerateFromEmptySnap(t *testing.T) {
helper.addTrieAccount(fmt.Sprintf("acc-%d", i),
&Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
}
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4
@ -735,6 +752,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) {
}
helper.addSnapStorage(accKey, moddedKeys, moddedVals)
}
root, snap := helper.CommitAndGenerate()
t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff
@ -759,6 +777,7 @@ func incKey(key []byte) []byte {
break
}
}
return key
}
@ -769,6 +788,7 @@ func decKey(key []byte) []byte {
break
}
}
return key
}

5
core/state/snapshot/holdable_iterator.go
View file

@ -42,6 +42,7 @@ func (it *holdableIterator) Hold() {
if it.it.Key() == nil {
return // nothing to hold
}
it.key = common.CopyBytes(it.it.Key())
it.val = common.CopyBytes(it.it.Value())
it.atHeld = false
@ -57,9 +58,11 @@ func (it *holdableIterator) Next() bool {
it.key = nil
it.val = nil
}
if it.key != nil {
return true // shifted to locally held value
}
return it.it.Next()
}
@ -83,6 +86,7 @@ func (it *holdableIterator) Key() []byte {
if it.key != nil {
return it.key
}
return it.it.Key()
}
@ -93,5 +97,6 @@ func (it *holdableIterator) Value() []byte {
if it.val != nil {
return it.val
}
return it.it.Value()
}

19
core/state/snapshot/holdable_iterator_test.go
View file

@ -33,6 +33,7 @@ func TestIteratorHold(t *testing.T) {
order = []string{"k1", "k2", "k3"}
db = rawdb.NewMemoryDatabase()
)
for key, val := range content {
if err := db.Put([]byte(key), []byte(val)); err != nil {
t.Fatalf("failed to insert item %s:%s into database: %v", key, val, err)
@ -49,9 +50,11 @@ func TestIteratorHold(t *testing.T) {
t.Errorf("more items than expected: checking idx=%d (key %q), expecting len=%d", idx, it.Key(), len(order))
break
}
if !bytes.Equal(it.Key(), []byte(order[idx])) {
t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx])
}
if !bytes.Equal(it.Value(), []byte(content[order[idx]])) {
t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]])
}
@ -61,9 +64,11 @@ func TestIteratorHold(t *testing.T) {
// Shift iterator to the discarded element
it.Next()
if !bytes.Equal(it.Key(), []byte(order[idx])) {
t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx])
}
if !bytes.Equal(it.Value(), []byte(content[order[idx]])) {
t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]])
}
@ -71,20 +76,25 @@ func TestIteratorHold(t *testing.T) {
// Discard/Next combo should work always
it.Hold()
it.Next()
if !bytes.Equal(it.Key(), []byte(order[idx])) {
t.Errorf("item %d: key mismatch: have %s, want %s", idx, string(it.Key()), order[idx])
}
if !bytes.Equal(it.Value(), []byte(content[order[idx]])) {
t.Errorf("item %d: value mismatch: have %s, want %s", idx, string(it.Value()), content[order[idx]])
}
idx++
}
if err := it.Error(); err != nil {
t.Errorf("iteration failed: %v", err)
}
if idx != len(order) {
t.Errorf("iteration terminated prematurely: have %d, want %d", idx, len(order))
}
db.Close()
}
@ -110,13 +120,16 @@ func TestReopenIterator(t *testing.T) {
}
db = rawdb.NewMemoryDatabase()
)
for key, val := range content {
rawdb.WriteAccountSnapshot(db, key, []byte(val))
}
checkVal := func(it *holdableIterator, index int) {
if !bytes.Equal(it.Key(), append(rawdb.SnapshotAccountPrefix, order[index].Bytes()...)) {
t.Fatalf("Unexpected data entry key, want %v got %v", order[index], it.Key())
}
if !bytes.Equal(it.Value(), []byte(content[order[index]])) {
t.Fatalf("Unexpected data entry key, want %v got %v", []byte(content[order[index]]), it.Value())
}
@ -125,11 +138,13 @@ func TestReopenIterator(t *testing.T) {
ctx, idx := newGeneratorContext(&generatorStats{}, db, nil, nil), -1
idx++
ctx.account.Next()
checkVal(ctx.account, idx)
ctx.reopenIterator(snapAccount)
idx++
ctx.account.Next()
checkVal(ctx.account, idx)
@ -137,6 +152,7 @@ func TestReopenIterator(t *testing.T) {
ctx.reopenIterator(snapAccount)
ctx.reopenIterator(snapAccount)
idx++
ctx.account.Next()
checkVal(ctx.account, idx)
@ -145,6 +161,7 @@ func TestReopenIterator(t *testing.T) {
ctx.account.Hold()
ctx.reopenIterator(snapAccount)
idx++
ctx.account.Next()
checkVal(ctx.account, idx)
@ -154,6 +171,7 @@ func TestReopenIterator(t *testing.T) {
ctx.reopenIterator(snapAccount)
ctx.reopenIterator(snapAccount)
idx++
ctx.account.Next()
checkVal(ctx.account, idx)
@ -161,6 +179,7 @@ func TestReopenIterator(t *testing.T) {
ctx.account.Next() // the end
ctx.reopenIterator(snapAccount)
ctx.account.Next()
if ctx.account.Key() != nil {
t.Fatal("Unexpected iterated entry")
}

17
core/state/snapshot/journal.go
View file

@ -109,6 +109,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou
// is not matched with disk layer; or the it's the legacy-format journal,
// etc.), we just discard all diffs and try to recover them later.
var current snapshot = base
err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error {
current = newDiffLayer(current, root, destructSet, accountData, storageData)
return nil
@ -116,6 +117,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou
if err != nil {
return base, generator, nil
}
return current, generator, nil
}
@ -285,6 +287,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
log.Warn("Loaded snapshot journal", "diffs", "missing")
return nil
}
r := rlp.NewStream(bytes.NewReader(journal), 0)
// Firstly, resolve the first element as the journal version
version, err := r.Uint64()
@ -292,6 +295,7 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
log.Warn("Failed to resolve the journal version", "error", err)
return errors.New("failed to resolve journal version")
}
if version != journalVersion {
log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version)
return errors.New("wrong journal version")
@ -303,10 +307,12 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
if err := r.Decode(&parent); err != nil {
return errors.New("missing disk layer root")
}
if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent {
log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched")
return fmt.Errorf("mismatched disk and diff layers")
}
for {
var (
root common.Hash
@ -323,20 +329,26 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
if errors.Is(err, io.EOF) {
return nil
}
return fmt.Errorf("load diff root: %v", err)
}
if err := r.Decode(&destructs); err != nil {
return fmt.Errorf("load diff destructs: %v", err)
}
if err := r.Decode(&accounts); err != nil {
return fmt.Errorf("load diff accounts: %v", err)
}
if err := r.Decode(&storage); err != nil {
return fmt.Errorf("load diff storage: %v", err)
}
for _, entry := range destructs {
destructSet[entry.Hash] = struct{}{}
}
for _, entry := range accounts {
if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
accountData[entry.Hash] = entry.Blob
@ -344,8 +356,10 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
accountData[entry.Hash] = nil
}
}
for _, entry := range storage {
slots := make(map[common.Hash][]byte)
for i, key := range entry.Keys {
if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that
slots[key] = entry.Vals[i]
@ -353,11 +367,14 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error {
slots[key] = nil
}
}
storageData[entry.Hash] = slots
}
if err := callback(parent, root, destructSet, accountData, storageData); err != nil {
return err
}
parent = root
}
}

1
core/state/snapshot/snapshot.go
View file

@ -211,6 +211,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root
}
if err != nil {
log.Warn("Failed to load snapshot", "err", err)
if !config.NoBuild {
snap.Rebuild(root)
return snap, nil

17
core/state/snapshot/utils.go
View file

@ -34,6 +34,7 @@ func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error {
if err := checkDanglingDiskStorage(chaindb); err != nil {
log.Error("Database check error", "err", err)
}
return checkDanglingMemStorage(chaindb)
}
@ -46,27 +47,34 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error {
lastKey []byte
it = rawdb.NewKeyLengthIterator(chaindb.NewIterator(rawdb.SnapshotStoragePrefix, nil), 1+2*common.HashLength)
)
log.Info("Checking dangling snapshot disk storage")
defer it.Release()
for it.Next() {
k := it.Key()
accKey := k[1:33]
if bytes.Equal(accKey, lastKey) {
// No need to look up for every slot
continue
}
lastKey = common.CopyBytes(accKey)
if time.Since(lastReport) > time.Second*8 {
log.Info("Iterating snap storage", "at", fmt.Sprintf("%#x", accKey), "elapsed", common.PrettyDuration(time.Since(start)))
lastReport = time.Now()
}
if data := rawdb.ReadAccountSnapshot(chaindb, common.BytesToHash(accKey)); len(data) == 0 {
log.Warn("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accKey), "storagekey", fmt.Sprintf("%#x", k))
return fmt.Errorf("dangling snapshot storage account %#x", accKey)
}
}
log.Info("Verified the snapshot disk storage", "time", common.PrettyDuration(time.Since(start)), "err", it.Error())
return nil
}
@ -74,7 +82,9 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error {
// snapshot difflayers.
func checkDanglingMemStorage(db ethdb.KeyValueStore) error {
start := time.Now()
log.Info("Checking dangling journalled storage")
err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {
for accHash := range storage {
if _, ok := accounts[accHash]; !ok {
@ -83,11 +93,14 @@ func checkDanglingMemStorage(db ethdb.KeyValueStore) error {
}
return nil
})
if err != nil {
log.Info("Failed to resolve snapshot journal", "err", err)
return err
}
log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start)))
return nil
}
@ -97,11 +110,14 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error {
// Look up the disk layer first
baseRoot := rawdb.ReadSnapshotRoot(db)
fmt.Printf("Disklayer: Root: %x\n", baseRoot)
if data := rawdb.ReadAccountSnapshot(db, hash); data != nil {
account := new(Account)
if err := rlp.DecodeBytes(data, account); err != nil {
panic(err)
}
fmt.Printf("\taccount.nonce: %d\n", account.Nonce)
fmt.Printf("\taccount.balance: %x\n", account.Balance)
fmt.Printf("\taccount.root: %x\n", account.Root)
@ -117,6 +133,7 @@ func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error {
}
it.Release()
}
var depth = 0
return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error {

13
core/state/state_object.go
View file

@ -146,9 +146,11 @@ func (s *stateObject) getTrie(db Database) (Trie, error) {
if err != nil {
return nil, err
}
s.trie = tr
}
}
return s.trie, nil
}
@ -197,10 +199,12 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has
if s.db.snap == nil || err != nil {
start := time.Now()
tr, err := s.getTrie(db)
if err != nil {
s.db.setError(err)
return common.Hash{}
}
enc, err = tr.GetStorage(s.address, key.Bytes())
if metrics.EnabledExpensive {
s.db.StorageReads += time.Since(start)
@ -252,6 +256,7 @@ func (s *stateObject) finalise(prefetch bool) {
slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure
}
}
if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash {
s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, slotsToPrefetch)
}
@ -278,7 +283,9 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) {
storage map[common.Hash][]byte
hasher = s.db.hasher
)
tr, err := s.getTrie(db)
if err != nil {
s.db.setError(err)
return nil, err
@ -327,6 +334,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) {
if len(s.pendingStorage) > 0 {
s.pendingStorage = make(Storage)
}
return tr, nil
}
@ -345,6 +353,7 @@ func (s *stateObject) updateRoot(db Database) {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now())
}
s.data.Root = tr.Hash()
}
@ -364,8 +373,10 @@ func (s *stateObject) commitTrie(db Database) (*trie.NodeSet, error) {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
}
root, nodes := tr.Commit(false)
s.data.Root = root
return nodes, nil
}
@ -433,6 +444,7 @@ func (s *stateObject) Code(db Database) []byte {
if s.code != nil {
return s.code
}
if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) {
return nil
}
@ -451,6 +463,7 @@ func (s *stateObject) CodeSize(db Database) int {
if s.code != nil {
return len(s.code)
}
if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) {
return 0
}

5
core/state/statedb.go
View file

@ -662,11 +662,14 @@ func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte,
if trie == nil {
return nil, errors.New("storage trie for requested address does not exist")
}
var proof proofList
err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
if err != nil {
return nil, err
}
return proof, nil
}
@ -696,6 +699,7 @@ func (s *StateDB) StorageTrie(addr common.Address) (Trie, error) {
return nil, nil
}
cpy := stateObject.deepCopy(s)
if _, err := cpy.updateTrie(s.db); err != nil {
return nil, err
}
@ -794,6 +798,7 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common
// to a previous incarnation of the object.
s.stateObjectsDestruct[addr] = struct{}{}
stateObject := s.GetOrNewStateObject(addr)
for k, v := range storage {
stateObject.SetState(s.db, k, v)
}

1
core/state/statedb_test.go
View file

@ -1396,6 +1396,7 @@ func TestFlushOrderDataLoss(t *testing.T) {
for a := byte(0); a < 10; a++ {
state.CreateAccount(common.Address{a})
for s := byte(0); s < 10; s++ {
state.SetState(common.Address{a}, common.Hash{a, s}, common.Hash{a, s})
}

3
core/state/sync_test.go
View file

@ -111,6 +111,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error {
if err != nil {
return err
}
it := t.NodeIterator(nil)
for it.Next(true) {
}
@ -189,6 +190,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) {
nodeElements []stateElement
codeElements []stateElement
)
paths, nodes, codes := sched.Missing(count)
for i := 0; i < len(paths); i++ {
@ -559,6 +561,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
if err != nil {
t.Fatalf("failed to retrieve node data for %x", element.hash)
}
results = append(results, trie.NodeSyncResult{Path: path, Data: data})
if len(results) >= cap(results) {

1
core/types/hashing_test.go
View file

@ -222,6 +222,7 @@ func (d *hashToHumanReadable) Reset() {
func (d *hashToHumanReadable) Update(i []byte, i2 []byte) error {
l := fmt.Sprintf("%x %x\n", i, i2)
d.data = append(d.data, []byte(l)...)
return nil
}

1
core/vm/instructions_test.go
View file

@ -250,6 +250,7 @@ func TestWriteExpectedValues(t *testing.T) {
pc = uint64(0)
interpreter = env.interpreter
)
result := make([]TwoOperandTestcase, len(args))
for i, param := range args {

1
tests/fuzzers/rangeproof/debug/main.go
View file

@ -32,6 +32,7 @@ func main() {
}
crasher := os.Args[1]
data, err := os.ReadFile(crasher)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err)
os.Exit(1)

1
tests/fuzzers/stacktrie/debug/main.go
View file

@ -30,6 +30,7 @@ func main() {
}
crasher := os.Args[1]
data, err := os.ReadFile(crasher)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err)
os.Exit(1)

10
trie/iterator_test.go
View file

@ -60,6 +60,7 @@ func TestIterator(t *testing.T) {
all[val.k] = val.v
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -89,6 +90,7 @@ func TestIteratorLargeData(t *testing.T) {
for i := byte(0); i < 255; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
trie.MustUpdate(value.k, value.v)
trie.MustUpdate(value2.k, value2.v)
vals[string(value.k)] = value
@ -222,6 +224,7 @@ func TestDifferenceIterator(t *testing.T) {
for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v))
}
rootA, nodesA := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
triea, _ = New(TrieID(rootA), dba)
@ -231,6 +234,7 @@ func TestDifferenceIterator(t *testing.T) {
for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
}
rootB, nodesB := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
trieb, _ = New(TrieID(rootB), dbb)
@ -264,6 +268,7 @@ func TestUnionIterator(t *testing.T) {
for _, val := range testdata1 {
triea.MustUpdate([]byte(val.k), []byte(val.v))
}
rootA, nodesA := triea.Commit(false)
dba.Update(NewWithNodeSet(nodesA))
triea, _ = New(TrieID(rootA), dba)
@ -273,6 +278,7 @@ func TestUnionIterator(t *testing.T) {
for _, val := range testdata2 {
trieb.MustUpdate([]byte(val.k), []byte(val.v))
}
rootB, nodesB := trieb.Commit(false)
dbb.Update(NewWithNodeSet(nodesB))
trieb, _ = New(TrieID(rootB), dbb)
@ -331,6 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
tr.MustUpdate([]byte(val.k), []byte(val.v))
}
_, nodes := tr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
if !memonly {
@ -423,6 +430,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
ctr.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := ctr.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
if !memonly {
@ -542,6 +550,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
val = crypto.Keccak256(val)
trie.MustUpdate(key, val)
}
_, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
// Return the generated trie
@ -582,6 +591,7 @@ func TestIteratorNodeBlob(t *testing.T) {
all[val.k] = val.v
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
_, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
triedb.Cap(0)

3
trie/node_test.go
View file

@ -104,6 +104,7 @@ func BenchmarkEncodeShortNode(b *testing.B) {
Key: []byte{0x1, 0x2},
Val: hashNode(randBytes(32)),
}
b.ResetTimer()
b.ReportAllocs()
@ -182,6 +183,7 @@ func BenchmarkDecodeFullNode(b *testing.B) {
for i := 0; i < 16; i++ {
node.Children[i] = hashNode(randBytes(32))
}
blob := nodeToBytes(node)
hash := crypto.Keccak256(blob)
@ -203,6 +205,7 @@ func BenchmarkDecodeFullNodeUnsafe(b *testing.B) {
for i := 0; i < 16; i++ {
node.Children[i] = hashNode(randBytes(32))
}
blob := nodeToBytes(node)
hash := crypto.Keccak256(blob)

6
trie/preimages.go
View file

@ -51,6 +51,7 @@ func (store *preimageStore) insertPreimage(preimages map[common.Hash][]byte) {
if _, ok := store.preimages[hash]; ok {
continue
}
store.preimages[hash] = preimage
store.preimagesSize += common.StorageSize(common.HashLength + len(preimage))
}
@ -66,6 +67,7 @@ func (store *preimageStore) preimage(hash common.Hash) []byte {
if preimage != nil {
return preimage
}
return rawdb.ReadPreimage(store.disk, hash)
}
@ -77,12 +79,16 @@ func (store *preimageStore) commit(force bool) error {
if store.preimagesSize <= 4*1024*1024 && !force {
return nil
}
batch := store.disk.NewBatch()
rawdb.WritePreimages(batch, store.preimages)
if err := batch.Write(); err != nil {
return err
}
store.preimages, store.preimagesSize = make(map[common.Hash][]byte), 0
return nil
}

3
trie/proof_test.go
View file

@ -40,12 +40,14 @@ func initRnd() *mrand.Rand {
crand.Read(seed[:])
rnd := mrand.New(mrand.NewSource(int64(binary.LittleEndian.Uint64(seed[:]))))
fmt.Printf("Seed: %x\n", seed)
return rnd
}
func randBytes(n int) []byte {
r := make([]byte, n)
prng.Read(r)
return r
}
@ -1047,6 +1049,7 @@ func randomTrie(n int) (*Trie, map[string]*kv) {
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
value2 := &kv{common.LeftPadBytes([]byte{i + 10}, 32), []byte{i}, false}
trie.MustUpdate(value.k, value.v)
trie.MustUpdate(value2.k, value2.v)
vals[string(value.k)] = value

17
trie/secure_trie.go
View file

@ -34,6 +34,7 @@ func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *D
Owner: owner,
Root: root,
}
return NewStateTrie(id, db)
}
@ -64,10 +65,12 @@ func NewStateTrie(id *ID, db *Database) (*StateTrie, error) {
if db == nil {
panic("trie.NewStateTrie called without a database")
}
trie, err := New(id, db)
if err != nil {
return nil, err
}
return &StateTrie{trie: *trie, preimages: db.preimages}, nil
}
@ -96,8 +99,10 @@ func (t *StateTrie) GetAccount(address common.Address) (*types.StateAccount, err
if res == nil || err != nil {
return nil, err
}
ret := new(types.StateAccount)
err = rlp.DecodeBytes(res, ret)
return ret, err
}
@ -109,8 +114,10 @@ func (t *StateTrie) GetAccountByHash(addrHash common.Hash) (*types.StateAccount,
if res == nil || err != nil {
return nil, err
}
ret := new(types.StateAccount)
err = rlp.DecodeBytes(res, ret)
return ret, err
}
@ -134,6 +141,7 @@ func (t *StateTrie) GetNode(path []byte) ([]byte, int, error) {
func (t *StateTrie) MustUpdate(key, value []byte) {
hk := t.hashKey(key)
t.trie.MustUpdate(hk, value)
t.getSecKeyCache()[string(hk)] = common.CopyBytes(key)
}
@ -159,13 +167,17 @@ func (t *StateTrie) UpdateStorage(_ common.Address, key, value []byte) error {
func (t *StateTrie) UpdateAccount(address common.Address, acc *types.StateAccount) error {
hk := t.hashKey(address.Bytes())
data, err := rlp.EncodeToBytes(acc)
if err != nil {
return err
}
if err := t.trie.Update(hk, data); err != nil {
return err
}
t.getSecKeyCache()[string(hk)] = address.Bytes()
return nil
}
@ -183,6 +195,7 @@ func (t *StateTrie) MustDelete(key []byte) {
func (t *StateTrie) DeleteStorage(_ common.Address, key []byte) error {
hk := t.hashKey(key)
delete(t.getSecKeyCache(), string(hk))
return t.trie.Delete(hk)
}
@ -190,6 +203,7 @@ func (t *StateTrie) DeleteStorage(_ common.Address, key []byte) error {
func (t *StateTrie) DeleteAccount(address common.Address) error {
hk := t.hashKey(address.Bytes())
delete(t.getSecKeyCache(), string(hk))
return t.trie.Delete(hk)
}
@ -199,9 +213,11 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
if t.preimages == nil {
return nil
}
return t.preimages.preimage(common.BytesToHash(shaKey))
}
@ -220,6 +236,7 @@ func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet) {
for hk, key := range t.secKeyCache {
preimages[common.BytesToHash([]byte(hk))] = key
}
t.preimages.insertPreimage(preimages)
}
t.secKeyCache = make(map[string][]byte)

2
trie/secure_trie_test.go
View file

@ -58,7 +58,9 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
trie.MustUpdate(key, val)
}
}
root, nodes := trie.Commit(false)
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}

5
trie/stacktrie.go
View file

@ -144,9 +144,11 @@ func (st *StackTrie) unmarshalBinary(r io.Reader) error {
Val []byte
Key []byte
}
if err := gob.NewDecoder(r).Decode(&dec); err != nil {
return err
}
st.owner = dec.Owner
st.nodeType = dec.NodeType
st.val = dec.Val
@ -160,6 +162,7 @@ func (st *StackTrie) unmarshalBinary(r io.Reader) error {
continue
}
var child StackTrie
if err := child.unmarshalBinary(r); err != nil {
return err
}
@ -208,6 +211,7 @@ func (st *StackTrie) Update(key, value []byte) error {
if len(value) == 0 {
panic("deletion not supported")
}
st.insert(k[:len(k)-1], value, nil)
return nil
}
@ -426,6 +430,7 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) {
nodes[i] = nilValueNode
continue
}
child.hashRec(hasher, append(path, byte(i)))
if len(child.val) < 32 {
nodes[i] = rawNode(child.val)

24
trie/sync.go
View file

@ -188,10 +188,13 @@ func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, par
if root == types.EmptyRootHash {
return
}
if s.membatch.hasNode(path) {
return
}
owner, inner := ResolvePath(path)
if rawdb.HasTrieNode(s.database, owner, inner, root, s.scheme) {
return
}
@ -210,6 +213,7 @@ func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, par
ancestor.deps++
req.parent = ancestor
}
s.scheduleNodeRequest(req)
}
@ -246,6 +250,7 @@ func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash, p
ancestor.deps++
req.parents = append(req.parents, ancestor)
}
s.scheduleCodeRequest(req)
}
@ -280,10 +285,12 @@ func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) {
log.Error("Missing node request", "path", item)
continue // System very wrong, shouldn't happen
}
nodePaths = append(nodePaths, item)
nodeHashes = append(nodeHashes, req.hash)
}
}
return nodePaths, nodeHashes, codeHashes
}
@ -299,10 +306,13 @@ func (s *Sync) ProcessCode(result CodeSyncResult) error {
if req == nil {
return ErrNotRequested
}
if req.data != nil {
return ErrAlreadyProcessed
}
req.data = result.Data
return s.commitCodeRequest(req)
}
@ -318,6 +328,7 @@ func (s *Sync) ProcessNode(result NodeSyncResult) error {
if req == nil {
return ErrNotRequested
}
if req.data != nil {
return ErrAlreadyProcessed
}
@ -326,6 +337,7 @@ func (s *Sync) ProcessNode(result NodeSyncResult) error {
if err != nil {
return err
}
req.data = result.Data
// Create and schedule a request for all the children nodes
@ -333,6 +345,7 @@ func (s *Sync) ProcessNode(result NodeSyncResult) error {
if err != nil {
return err
}
if len(requests) == 0 && req.deps == 0 {
s.commitNodeRequest(req)
} else {
@ -352,6 +365,7 @@ func (s *Sync) Commit(dbw ethdb.Batch) error {
owner, inner := ResolvePath([]byte(path))
rawdb.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value, s.scheme)
}
for hash, value := range s.membatch.codes {
rawdb.WriteCode(dbw, hash, value)
}
@ -394,6 +408,7 @@ func (s *Sync) scheduleCodeRequest(req *codeRequest) {
old.parents = append(old.parents, req.parents...)
return
}
s.codeReqs[req.hash] = req
// Schedule the request for future retrieval. This queue is shared
@ -413,6 +428,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
path []byte
node node
}
var children []childNode
switch node := (object).(type) {
@ -421,6 +437,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
if hasTerm(key) {
key = key[:len(key)-1]
}
children = []childNode{{
node: node.Val,
path: append(append([]byte(nil), req.path...), key...),
@ -453,6 +470,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength]))
paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:]))
}
if err := req.callback(paths, child.path, node, req.hash, req.path); err != nil {
return nil, err
}
@ -466,6 +484,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
}
// Check the presence of children concurrently
pending.Add(1)
go func(child childNode) {
defer pending.Done()
@ -475,6 +494,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
chash = common.BytesToHash(node)
owner, inner = ResolvePath(child.path)
)
if rawdb.HasTrieNode(s.database, owner, inner, chash, s.scheme) {
return
}
@ -488,9 +508,11 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
}(child)
}
}
pending.Wait()
requests := make([]*nodeRequest, 0, len(children))
for done := false; !done; {
select {
case miss := <-missing:
@ -525,6 +547,7 @@ func (s *Sync) commitNodeRequest(req *nodeRequest) error {
}
}
}
return nil
}
@ -558,5 +581,6 @@ func ResolvePath(path []byte) (common.Hash, []byte) {
owner = common.BytesToHash(hexToKeybytes(path[:2*common.HashLength]))
path = path[2*common.HashLength:]
}
return owner, path
}

40
trie/sync_test.go
View file

@ -53,7 +53,9 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) {
trie.MustUpdate(key, val)
}
}
root, nodes := trie.Commit(false)
if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
panic(fmt.Errorf("failed to commit db %v", err))
}
@ -135,6 +137,7 @@ func testIterativeSync(t *testing.T, count int, bypath bool) {
// at the testing trie.
paths, nodes, _ := sched.Missing(count)
var elements []trieElement
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -142,14 +145,17 @@ func testIterativeSync(t *testing.T, count int, bypath bool) {
syncPath: NewSyncPath([]byte(paths[i])),
})
}
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
if !bypath {
for i, element := range elements {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err)
}
results[i] = NodeSyncResult{element.path, data}
}
} else {
@ -174,6 +180,7 @@ func testIterativeSync(t *testing.T, count int, bypath bool) {
paths, nodes, _ = sched.Missing(count)
elements = elements[:0]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -201,6 +208,7 @@ func TestIterativeDelayedSync(t *testing.T) {
// at the testing trie.
paths, nodes, _ := sched.Missing(10000)
var elements []trieElement
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -208,6 +216,7 @@ func TestIterativeDelayedSync(t *testing.T) {
syncPath: NewSyncPath([]byte(paths[i])),
})
}
for len(elements) > 0 {
// Sync only half of the scheduled nodes
results := make([]NodeSyncResult, len(elements)/2+1)
@ -216,6 +225,7 @@ func TestIterativeDelayedSync(t *testing.T) {
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
results[i] = NodeSyncResult{element.path, data}
}
for _, result := range results {
@ -231,6 +241,7 @@ func TestIterativeDelayedSync(t *testing.T) {
paths, nodes, _ = sched.Missing(10000)
elements = elements[len(results):]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -262,6 +273,7 @@ func testIterativeRandomSync(t *testing.T, count int) {
// at the testing trie.
paths, nodes, _ := sched.Missing(count)
queue := make(map[string]trieElement)
for i, path := range paths {
queue[path] = trieElement{
path: paths[i],
@ -272,11 +284,13 @@ func testIterativeRandomSync(t *testing.T, count int) {
for len(queue) > 0 {
// Fetch all the queued nodes in a random order
results := make([]NodeSyncResult, 0, len(queue))
for path, element := range queue {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
results = append(results, NodeSyncResult{path, data})
}
// Feed the retrieved results back and queue new tasks
@ -293,6 +307,7 @@ func testIterativeRandomSync(t *testing.T, count int) {
paths, nodes, _ = sched.Missing(count)
queue = make(map[string]trieElement)
for i, path := range paths {
queue[path] = trieElement{
path: path,
@ -320,6 +335,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
// at the testing trie.
paths, nodes, _ := sched.Missing(10000)
queue := make(map[string]trieElement)
for i, path := range paths {
queue[path] = trieElement{
path: path,
@ -330,11 +346,13 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
for len(queue) > 0 {
// Sync only half of the scheduled nodes, even those in random order
results := make([]NodeSyncResult, 0, len(queue)/2+1)
for path, element := range queue {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
results = append(results, NodeSyncResult{path, data})
if len(results) >= cap(results) {
@ -355,7 +373,9 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
for _, result := range results {
delete(queue, result.Path)
}
paths, nodes, _ = sched.Missing(10000)
for i, path := range paths {
queue[path] = trieElement{
path: path,
@ -383,6 +403,7 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
// at the testing trie.
paths, nodes, _ := sched.Missing(0)
var elements []trieElement
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -394,14 +415,17 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
if _, ok := requested[element.hash]; ok {
t.Errorf("hash %x already requested once", element.hash)
}
requested[element.hash] = struct{}{}
results[i] = NodeSyncResult{element.path, data}
@ -419,6 +443,7 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
paths, nodes, _ = sched.Missing(0)
elements = elements[:0]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -451,6 +476,7 @@ func TestIncompleteSync(t *testing.T) {
root = srcTrie.Hash()
)
paths, nodes, _ := sched.Missing(1)
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -458,14 +484,17 @@ func TestIncompleteSync(t *testing.T) {
syncPath: NewSyncPath([]byte(paths[i])),
})
}
for len(elements) > 0 {
// Fetch a batch of trie nodes
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
results[i] = NodeSyncResult{element.path, data}
}
// Process each of the trie nodes
@ -493,6 +522,7 @@ func TestIncompleteSync(t *testing.T) {
// Fetch the next batch to retrieve
paths, nodes, _ = sched.Missing(1)
elements = elements[:0]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
@ -505,6 +535,7 @@ func TestIncompleteSync(t *testing.T) {
for _, hash := range added {
value, _ := diskdb.Get(hash.Bytes())
diskdb.Delete(hash.Bytes())
if err := checkTrieConsistency(triedb, root); err == nil {
t.Fatalf("trie inconsistency not caught, missing: %x", hash)
}
@ -529,23 +560,28 @@ func TestSyncOrdering(t *testing.T) {
reqs []SyncPath
elements []trieElement
)
paths, nodes, _ := sched.Missing(1)
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
hash: nodes[i],
syncPath: NewSyncPath([]byte(paths[i])),
})
reqs = append(reqs, NewSyncPath([]byte(paths[i])))
}
for len(elements) > 0 {
results := make([]NodeSyncResult, len(elements))
for i, element := range elements {
data, err := srcDb.Node(element.hash)
if err != nil {
t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err)
}
results[i] = NodeSyncResult{element.path, data}
}
for _, result := range results {
@ -561,16 +597,18 @@ func TestSyncOrdering(t *testing.T) {
paths, nodes, _ = sched.Missing(1)
elements = elements[:0]
for i := 0; i < len(paths); i++ {
elements = append(elements, trieElement{
path: paths[i],
hash: nodes[i],
syncPath: NewSyncPath([]byte(paths[i])),
})
reqs = append(reqs, NewSyncPath([]byte(paths[i])))
}
}
// Cross check that the two tries are in sync
// Cross-check that the two tries are in sync
checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData)
// Check that the trie nodes have been requested path-ordered

7
trie/tracer.go
View file

@ -67,6 +67,7 @@ func (t *tracer) onInsert(path []byte) {
delete(t.deletes, string(path))
return
}
t.inserts[string(path)] = struct{}{}
}
@ -78,6 +79,7 @@ func (t *tracer) onDelete(path []byte) {
delete(t.inserts, string(path))
return
}
t.deletes[string(path)] = struct{}{}
}
@ -95,15 +97,19 @@ func (t *tracer) copy() *tracer {
deletes = make(map[string]struct{})
accessList = make(map[string][]byte)
)
for path := range t.inserts {
inserts[path] = struct{}{}
}
for path := range t.deletes {
deletes[path] = struct{}{}
}
for path, blob := range t.accessList {
accessList[path] = common.CopyBytes(blob)
}
return &tracer{
inserts: inserts,
deletes: deletes,
@ -120,6 +126,7 @@ func (t *tracer) markDeletions(set *NodeSet) {
if _, ok := set.accessList[path]; !ok {
continue
}
set.markDeleted([]byte(path))
}
}

41
trie/tracer_test.go
View file

@ -69,6 +69,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
for _, val := range vals {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
insertSet := copySet(trie.tracer.inserts) // copy before commit
deleteSet := copySet(trie.tracer.deletes) // copy before commit
root, nodes := trie.Commit(false)
@ -78,6 +79,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
if !compareSet(insertSet, seen) {
t.Fatal("Unexpected insertion set")
}
if !compareSet(deleteSet, nil) {
t.Fatal("Unexpected deletion set")
}
@ -87,10 +89,13 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) {
for _, val := range vals {
trie.MustDelete([]byte(val.k))
}
insertSet, deleteSet = copySet(trie.tracer.inserts), copySet(trie.tracer.deletes)
if !compareSet(insertSet, nil) {
t.Fatal("Unexpected insertion set")
}
if !compareSet(deleteSet, seen) {
t.Fatal("Unexpected deletion set")
}
@ -112,12 +117,15 @@ func testTrieTracerNoop(t *testing.T, vals []struct{ k, v string }) {
for _, val := range vals {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
for _, val := range vals {
trie.MustDelete([]byte(val.k))
}
if len(trie.tracer.inserts) != 0 {
t.Fatal("Unexpected insertion set")
}
if len(trie.tracer.deletes) != 0 {
t.Fatal("Unexpected deletion set")
}
@ -143,6 +151,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
for _, val := range vals {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -154,9 +163,11 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Update trie
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, val := range vals {
trie.MustUpdate([]byte(val.k), randBytes(32))
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -169,11 +180,13 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
var keys []string
for i := 0; i < 30; i++ {
key := randBytes(32)
keys = append(keys, string(key))
trie.MustUpdate(key, randBytes(32))
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -185,9 +198,11 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Partial deletions
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, key := range keys {
trie.MustUpdate([]byte(key), nil)
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -199,9 +214,11 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Delete all
trie, _ = New(TrieID(root), db)
orig = trie.Copy()
for _, val := range vals {
trie.MustUpdate([]byte(val.k), nil)
}
root, nodes = trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -214,6 +231,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) {
// Tests origin values won't be tracked in Iterator or Prover
func TestAccessListLeak(t *testing.T) {
t.Parallel()
var (
db = NewDatabase(rawdb.NewMemoryDatabase())
trie = NewEmpty(db)
@ -222,6 +240,7 @@ func TestAccessListLeak(t *testing.T) {
for _, val := range standard {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
@ -250,6 +269,7 @@ func TestAccessListLeak(t *testing.T) {
},
},
}
for _, c := range cases {
trie, _ = New(TrieID(root), db)
n1 := len(trie.tracer.accessList)
@ -266,21 +286,26 @@ func TestAccessListLeak(t *testing.T) {
// in its parent due to the smaller size of the original tree node.
func TestTinyTree(t *testing.T) {
t.Parallel()
var (
db = NewDatabase(rawdb.NewMemoryDatabase())
trie = NewEmpty(db)
)
for _, val := range tiny {
trie.MustUpdate([]byte(val.k), randBytes(32))
}
root, set := trie.Commit(false)
db.Update(NewWithNodeSet(set))
trie, _ = New(TrieID(root), db)
orig := trie.Copy()
for _, val := range tiny {
trie.MustUpdate([]byte(val.k), []byte(val.v))
}
root, set = trie.Commit(false)
db.Update(NewWithNodeSet(set))
@ -294,11 +319,13 @@ func compareSet(setA, setB map[string]struct{}) bool {
if len(setA) != len(setB) {
return false
}
for key := range setA {
if _, ok := setB[key]; !ok {
return false
}
}
return true
}
@ -307,12 +334,15 @@ func forNodes(tr *Trie) map[string][]byte {
it = tr.NodeIterator(nil)
nodes = make(map[string][]byte)
)
for it.Next(true) {
if it.Leaf() {
continue
}
nodes[string(it.Path())] = common.CopyBytes(it.NodeBlob())
}
return nodes
}
@ -326,12 +356,15 @@ func forHashedNodes(tr *Trie) map[string][]byte {
it = tr.NodeIterator(nil)
nodes = make(map[string][]byte)
)
for it.Next(true) {
if it.Hash() == (common.Hash{}) {
continue
}
nodes[string(it.Path())] = common.CopyBytes(it.NodeBlob())
}
return nodes
}
@ -343,22 +376,28 @@ func diffTries(trieA, trieB *Trie) (map[string][]byte, map[string][]byte, map[st
inB = make(map[string][]byte) // hashed nodes in trie b but not a
both = make(map[string][]byte) // hashed nodes in both tries but different value
)
for path, blobA := range nodesA {
if blobB, ok := nodesB[path]; ok {
if bytes.Equal(blobA, blobB) {
continue
}
both[path] = blobA
continue
}
inA[path] = blobA
}
for path, blobB := range nodesB {
if _, ok := nodesA[path]; ok {
continue
}
inB[path] = blobB
}
return inA, inB, both
}
@ -367,6 +406,7 @@ func setKeys(set map[string][]byte) map[string]struct{} {
for k := range set {
keys[k] = struct{}{}
}
return keys
}
@ -375,5 +415,6 @@ func copySet(set map[string]struct{}) map[string]struct{} {
for k := range set {
copied[k] = struct{}{}
}
return copied
}

14
trie/trie.go
View file

@ -83,6 +83,7 @@ func New(id *ID, db NodeReader) (*Trie, error) {
reader: reader,
tracer: newTracer(),
}
if id.Root != (common.Hash{}) && id.Root != types.EmptyRootHash {
rootnode, err := trie.resolveAndTrack(id.Root[:], nil)
if err != nil {
@ -139,6 +140,7 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
// key not found in trie
return nil, n, false, nil
}
value, newnode, didResolve, err = t.get(n.Val, key, pos+len(n.Key))
if err == nil && didResolve {
n = n.copy()
@ -157,6 +159,7 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
if err != nil {
return nil, n, true, err
}
value, newnode, _, err := t.get(child, key, pos)
return value, newnode, true, err
default:
@ -171,6 +174,7 @@ func (t *Trie) MustGetNode(path []byte) ([]byte, int) {
if err != nil {
log.Error("Unhandled trie error in Trie.GetNode", "err", err)
}
return item, resolved
}
@ -190,6 +194,7 @@ func (t *Trie) GetNode(path []byte) ([]byte, int, error) {
if item == nil {
return nil, resolved, nil
}
return item, resolved, nil
}
@ -212,6 +217,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
if hash == nil {
return nil, origNode, 0, errors.New("non-consensus node")
}
blob, err := t.reader.nodeBlob(path, common.BytesToHash(hash))
return blob, origNode, 1, err
}
@ -226,6 +232,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
// Path branches off from short node
return nil, n, 0, nil
}
item, newnode, resolved, err = t.getNode(n.Val, path, pos+len(n.Key))
if err == nil && resolved > 0 {
n = n.copy()
@ -246,6 +253,7 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
if err != nil {
return nil, n, 1, err
}
item, newnode, resolved, err := t.getNode(child, path, pos)
return item, newnode, resolved + 1, err
@ -553,7 +561,9 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) {
if err != nil {
return nil, err
}
t.tracer.onRead(prefix, blob)
return mustDecodeNode(n, blob), nil
}
@ -595,7 +605,9 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet) {
t.root = hashedNode
return rootHash, nil
}
t.root = newCommitter(nodes, collectLeaf).Commit(t.root)
return rootHash, nodes
}
@ -608,9 +620,11 @@ func (t *Trie) hashRoot() (node, node) {
h := newHasher(t.unhashed >= 100)
defer func() {
returnHasherToPool(h)
t.unhashed = 0
}()
hashed, cached := h.hash(t.root, true)
return hashed, cached
}

9
trie/trie_reader.go
View file

@ -56,6 +56,7 @@ func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, er
if reader == nil {
return nil, fmt.Errorf("state not found #%x", stateRoot)
}
return &trieReader{owner: owner, reader: reader}, nil
}
@ -75,13 +76,17 @@ func (r *trieReader) node(path []byte, hash common.Hash) (node, error) {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
}
if r.reader == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
node, err := r.reader.Node(r.owner, path, hash)
if err != nil || node == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err}
}
return node, nil
}
@ -95,12 +100,16 @@ func (r *trieReader) nodeBlob(path []byte, hash common.Hash) ([]byte, error) {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
}
if r.reader == nil {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path}
}
blob, err := r.reader.NodeBlob(r.owner, path, hash)
if err != nil || len(blob) == 0 {
return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err}
}
return blob, nil
}

54
trie/trie_test.go
View file

@ -57,6 +57,7 @@ func TestNull(t *testing.T) {
key := make([]byte, 32)
value := []byte("test")
trie.MustUpdate(key, value)
if !bytes.Equal(trie.MustGet(key), value) {
t.Fatal("wrong value")
}
@ -94,21 +95,25 @@ func testMissingNode(t *testing.T, memonly bool) {
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("120099"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Update([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Delete([]byte("123456"))
if err != nil {
@ -127,21 +132,25 @@ func testMissingNode(t *testing.T, memonly bool) {
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("120099"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
_, err = trie.Get([]byte("123456"))
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Update([]byte("120099"), []byte("zxcv"))
if _, ok := err.(*MissingNodeError); !ok {
t.Errorf("Wrong error: %v", err)
}
trie, _ = New(TrieID(root), triedb)
err = trie.Delete([]byte("123456"))
if _, ok := err.(*MissingNodeError); !ok {
@ -191,6 +200,7 @@ func TestGet(t *testing.T) {
if i == 1 {
return
}
root, nodes := trie.Commit(false)
db.Update(NewWithNodeSet(nodes))
trie, _ = New(TrieID(root), db)
@ -263,6 +273,7 @@ func TestReplication(t *testing.T) {
for _, val := range vals {
updateString(trie, val.k, val.v)
}
exp, nodes := trie.Commit(false)
triedb.Update(NewWithNodeSet(nodes))
@ -276,6 +287,7 @@ func TestReplication(t *testing.T) {
t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
}
}
hash, nodes := trie2.Commit(false)
if hash != exp {
t.Errorf("root failure. expected %x got %x", exp, hash)
@ -285,7 +297,9 @@ func TestReplication(t *testing.T) {
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
}
trie2, err = New(TrieID(hash), triedb)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
@ -411,7 +425,9 @@ func verifyAccessList(oldTrie *Trie, newTrie *Trie, set *NodeSet) error {
if !ok || n.isDeleted() {
return errors.New("expect new node")
}
_, ok = set.accessList[path]
if ok {
return errors.New("unexpected origin value")
}
@ -422,10 +438,13 @@ func verifyAccessList(oldTrie *Trie, newTrie *Trie, set *NodeSet) error {
if !ok || !n.isDeleted() {
return errors.New("expect deleted node")
}
v, ok := set.accessList[path]
if !ok {
return errors.New("expect origin value")
}
if !bytes.Equal(v, blob) {
return errors.New("invalid origin value")
}
@ -436,14 +455,18 @@ func verifyAccessList(oldTrie *Trie, newTrie *Trie, set *NodeSet) error {
if !ok || n.isDeleted() {
return errors.New("expect updated node")
}
v, ok := set.accessList[path]
if !ok {
return errors.New("expect origin value")
}
if !bytes.Equal(v, blob) {
return errors.New("invalid origin value")
}
}
return nil
}
@ -476,12 +499,16 @@ func runRandTest(rt randTest) bool {
if hash == types.EmptyRootHash {
continue
}
proofDb := rawdb.NewMemoryDatabase()
err := tr.Prove(step.key, 0, proofDb)
if err != nil {
rt[i].err = fmt.Errorf("failed for proving key %#x, %v", step.key, err)
}
_, err = VerifyProof(hash, step.key, proofDb)
if err != nil {
rt[i].err = fmt.Errorf("failed for verifying key %#x, %v", step.key, err)
}
@ -492,11 +519,13 @@ func runRandTest(rt randTest) bool {
if nodes != nil {
triedb.Update(NewWithNodeSet(nodes))
}
newtr, err := New(TrieID(root), triedb)
if err != nil {
rt[i].err = err
return false
}
if nodes != nil {
if err := verifyAccessList(origTrie, newtr, nodes); err != nil {
rt[i].err = err
@ -521,45 +550,56 @@ func runRandTest(rt randTest) bool {
origSeen = make(map[string]struct{})
curSeen = make(map[string]struct{})
)
for origIter.Next(true) {
if origIter.Leaf() {
continue
}
origSeen[string(origIter.Path())] = struct{}{}
}
for curIter.Next(true) {
if curIter.Leaf() {
continue
}
curSeen[string(curIter.Path())] = struct{}{}
}
var (
insertExp = make(map[string]struct{})
deleteExp = make(map[string]struct{})
)
for path := range curSeen {
_, present := origSeen[path]
if !present {
insertExp[path] = struct{}{}
}
}
for path := range origSeen {
_, present := curSeen[path]
if !present {
deleteExp[path] = struct{}{}
}
}
if len(insertExp) != len(tr.tracer.inserts) {
rt[i].err = fmt.Errorf("insert set mismatch")
}
if len(deleteExp) != len(tr.tracer.deletes) {
rt[i].err = fmt.Errorf("delete set mismatch")
}
for insert := range tr.tracer.inserts {
if _, present := insertExp[insert]; !present {
rt[i].err = fmt.Errorf("missing inserted node")
}
}
for del := range tr.tracer.deletes {
if _, present := deleteExp[del]; !present {
rt[i].err = fmt.Errorf("missing deleted node")
@ -691,19 +731,24 @@ func TestTinyTrie(t *testing.T) {
if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root {
t.Errorf("1: got %x, exp %x", root, exp)
}
trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"), accounts[4])
if exp, root := common.HexToHash("ec63b967e98a5720e7f720482151963982890d82c9093c0d486b7eb8883a66b1"), trie.Hash(); exp != root {
t.Errorf("2: got %x, exp %x", root, exp)
}
trie.MustUpdate(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"), accounts[4])
if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root {
t.Errorf("3: got %x, exp %x", root, exp)
}
checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
checktr.MustUpdate(it.Key, it.Value)
}
if troot, itroot := trie.Hash(), checktr.Hash(); troot != itroot {
t.Fatalf("hash mismatch in opItercheckhash, trie: %x, check: %x", troot, itroot)
}
@ -721,10 +766,13 @@ func TestCommitAfterHash(t *testing.T) {
trie.Commit(false)
root := trie.Hash()
exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
root, _ = trie.Commit(false)
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
@ -1019,7 +1067,9 @@ func BenchmarkHashFixedSize(b *testing.B) {
func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -1070,7 +1120,9 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) {
func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@ -1122,8 +1174,10 @@ func BenchmarkDerefRootFixedSize(b *testing.B) {
func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) {
b.ReportAllocs()
triedb := NewDatabase(rawdb.NewMemoryDatabase())
trie := NewEmpty(triedb)
for i := 0; i < len(addresses); i++ {
trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i])
}