forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-19 21:31:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #925 from gzliudan/code-prefix

Browse source 
consensus, core, eth, trie: add a prefix to contract code
This commit is contained in:
Daniel Liu 2025-03-25 11:56:08 +08:00 committed by GitHub
commit cd975aa165
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
41 changed files with 562 additions and 295 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
XDCx/tradingstate/XDCx_trie.go
View file

@ -147,8 +147,7 @@ func (t *XDCXTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
key, _ := t.trie.Db.Preimage(common.BytesToHash(shaKey))
return key
return t.trie.Db.Preimage(common.BytesToHash(shaKey))
}
// Commit writes all nodes and the secure hash pre-images to the trie's database.

4
XDCxlending/lendingstate/XDCx_trie.go
View file

@ -18,6 +18,7 @@ package lendingstate
import (
"fmt"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/trie"
@ -142,8 +143,7 @@ func (t *XDCXTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
key, _ := t.trie.Db.Preimage(common.BytesToHash(shaKey))
return key
return t.trie.Db.Preimage(common.BytesToHash(shaKey))
}
// Commit writes all nodes and the secure hash pre-images to the trie's database.

3
consensus/XDPoS/engines/engine_v1/engine.go
View file

@ -25,6 +25,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
const (
@ -849,7 +850,7 @@ func (x *XDPoS_v1) Finalize(chain consensus.ChainReader, header *types.Header, s
header.UncleHash = types.CalcUncleHash(nil)
// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts), nil
return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil
}
// Authorize injects a private key into the consensus engine to mint new blocks

3
consensus/XDPoS/engines/engine_v2/engine.go
View file

@ -22,6 +22,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
type XDPoS_v2 struct {
@ -414,7 +415,7 @@ func (x *XDPoS_v2) Finalize(chain consensus.ChainReader, header *types.Header, s
header.UncleHash = types.CalcUncleHash(nil)
// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts), nil
return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil
}
// Authorize injects a private key into the consensus engine to mint new blocks with.

3
consensus/clique/clique.go
View file

@ -40,6 +40,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/rpc"
"github.com/XinFinOrg/XDPoSChain/trie"
"golang.org/x/crypto/sha3"
)
@ -588,7 +589,7 @@ func (c *Clique) Finalize(chain consensus.ChainReader, header *types.Header, sta
header.UncleHash = types.CalcUncleHash(nil)
// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts), nil
return types.NewBlock(header, txs, nil, receipts, new(trie.Trie)), nil
}
// Authorize injects a private key into the consensus engine to mint new blocks

3
consensus/ethash/consensus.go
View file

@ -32,6 +32,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/state"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
mapset "github.com/deckarep/golang-set/v2"
)
@ -523,7 +524,7 @@ func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header
header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number))
// Header seems complete, assemble into a block and return
return types.NewBlock(header, txs, uncles, receipts), nil
return types.NewBlock(header, txs, uncles, receipts, new(trie.Trie)), nil
}
// Some weird constants to avoid constant memory allocs for them.

3
consensus/tests/engine_v1_tests/helper.go
View file

@ -26,6 +26,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/trie"
)
type masterNodes map[string]big.Int
@ -390,7 +391,7 @@ func createBlockFromHeader(bc *core.BlockChain, customHeader *types.Header, txs
}
header.GasUsed = *gasUsed
block = types.NewBlock(&header, txs, nil, receipts)
block = types.NewBlock(&header, txs, nil, receipts, new(trie.Trie))
}
return block, nil

3
consensus/tests/engine_v2_tests/helper.go
View file

@ -29,6 +29,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/trie"
"github.com/stretchr/testify/assert"
)
@ -879,7 +880,7 @@ func createBlockFromHeader(bc *core.BlockChain, customHeader *types.Header, txs
header.Coinbase = signerAddress
sealHeader(bc, &header, signerAddress, signerFunction)
block = types.NewBlock(&header, txs, nil, receipts)
block = types.NewBlock(&header, txs, nil, receipts, new(trie.Trie))
}
return block, nil

5
core/block_validator.go
View file

@ -29,6 +29,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
// BlockValidator is responsible for validating block headers, uncles and
@ -67,7 +68,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
return fmt.Errorf("uncle root hash mismatch: have %x, want %x", hash, header.UncleHash)
}
if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
if hash := types.DeriveSha(block.Transactions(), new(trie.Trie)); hash != header.TxHash {
return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
}
if !v.bc.HasBlockAndFullState(block.ParentHash(), block.NumberU64()-1) {
@ -95,7 +96,7 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat
return fmt.Errorf("invalid bloom (remote: %x local: %x)", header.Bloom, rbloom)
}
// Tre receipt Trie's root (R = (Tr [[H1, R1], ... [Hn, R1]]))
receiptSha := types.DeriveSha(receipts)
receiptSha := types.DeriveSha(receipts, new(trie.Trie))
if receiptSha != header.ReceiptHash {
return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
}

25
core/blockchain.go
View file

@ -115,7 +115,10 @@ const (
// - Version 6
// The following incompatible database changes were added:
// * Transaction lookup information stores the corresponding block number instead of block hash
BlockChainVersion uint64 = 6
// - Version 7
// The following incompatible database changes were added:
// * New scheme for contract code in order to separate the codes and trie nodes
BlockChainVersion uint64 = 7
// Maximum length of chain to cache by block's number
blocksHashCacheLimit = 900
@ -995,12 +998,30 @@ func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.
return uncles
}
// TrieNode retrieves a blob of data associated with a trie node (or code hash)
// TrieNode retrieves a blob of data associated with a trie node
// either from ephemeral in-memory cache, or from persistent storage.
func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
return bc.stateCache.TrieDB().Node(hash)
}
// ContractCode retrieves a blob of data associated with a contract hash
// either from ephemeral in-memory cache, or from persistent storage.
func (bc *BlockChain) ContractCode(hash common.Hash) ([]byte, error) {
return bc.stateCache.ContractCode(common.Hash{}, hash)
}
// ContractCodeWithPrefix retrieves a blob of data associated with a contract
// hash either from ephemeral in-memory cache, or from persistent storage.
//
// If the code doesn't exist in the in-memory cache, check the storage with
// new code scheme.
func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) ([]byte, error) {
type codeReader interface {
ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error)
}
return bc.stateCache.(codeReader).ContractCodeWithPrefix(common.Hash{}, hash)
}
func (bc *BlockChain) saveData() {
// Ensure the state of a recent block is also stored to disk before exiting.
// We're writing three different states to catch different restart scenarios:

5
core/blockchain_test.go
View file

@ -35,6 +35,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
// newCanonical creates a chain database, and injects a deterministic canonical
@ -653,12 +654,12 @@ func TestFastVsFullChains(t *testing.T) {
}
if fblock, ablock := fast.GetBlockByHash(hash), archive.GetBlockByHash(hash); fblock.Hash() != ablock.Hash() {
t.Errorf("block #%d [%x]: block mismatch: have %v, want %v", num, hash, fblock, ablock)
} else if types.DeriveSha(fblock.Transactions()) != types.DeriveSha(ablock.Transactions()) {
} else if types.DeriveSha(fblock.Transactions(), new(trie.Trie)) != types.DeriveSha(ablock.Transactions(), new(trie.Trie)) {
t.Errorf("block #%d [%x]: transactions mismatch: have %v, want %v", num, hash, fblock.Transactions(), ablock.Transactions())
} else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) {
t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles())
}
if freceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash), fast.Config()), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) {
if freceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash), fast.Config()), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts, new(trie.Trie)) != types.DeriveSha(areceipts, new(trie.Trie)) {
t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts)
}
}

3
core/genesis.go
View file

@ -32,6 +32,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
//go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go
@ -228,7 +229,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block {
statedb.Commit(false)
statedb.Database().TrieDB().Commit(root, true)
return types.NewBlock(head, nil, nil, nil)
return types.NewBlock(head, nil, nil, nil, new(trie.Trie))
}
// Commit writes the block and state of a genesis specification to the database.

4
core/rawdb/accessors_chain_test.go
View file

@ -106,7 +106,7 @@ func TestBodyStorage(t *testing.T) {
WriteBody(db, hash, 0, body)
if entry := ReadBody(db, hash, 0); entry == nil {
t.Fatalf("Stored body not found")
} else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) {
} else if types.DeriveSha(types.Transactions(entry.Transactions), newHasher()) != types.DeriveSha(types.Transactions(body.Transactions), newHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) {
t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body)
}
if entry := ReadBodyRLP(db, hash, 0); entry == nil {
@ -160,7 +160,7 @@ func TestBlockStorage(t *testing.T) {
}
if entry := ReadBody(db, block.Hash(), block.NumberU64()); entry == nil {
t.Fatalf("Stored body not found")
} else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) {
} else if types.DeriveSha(types.Transactions(entry.Transactions), newHasher()) != types.DeriveSha(block.Transactions(), newHasher()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) {
t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, block.Body())
}
// Delete the block and verify the execution

28
core/rawdb/accessors_indexes_test.go
View file

@ -17,6 +17,7 @@
package rawdb
import (
"hash"
"math/big"
"testing"
@ -24,8 +25,33 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/rlp"
"golang.org/x/crypto/sha3"
)
// testHasher is the helper tool for transaction/receipt list hashing.
// The original hasher is trie, in order to get rid of import cycle,
// use the testing hasher instead.
type testHasher struct {
hasher hash.Hash
}
func newHasher() *testHasher {
return &testHasher{hasher: sha3.NewLegacyKeccak256()}
}
func (h *testHasher) Reset() {
h.hasher.Reset()
}
func (h *testHasher) Update(key, val []byte) {
h.hasher.Write(key)
h.hasher.Write(val)
}
func (h *testHasher) Hash() common.Hash {
return common.BytesToHash(h.hasher.Sum(nil))
}
// Tests that positional lookup metadata can be stored and retrieved.
func TestLookupStorage(t *testing.T) {
tests := []struct {
@ -71,7 +97,7 @@ func TestLookupStorage(t *testing.T) {
tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
txs := []*types.Transaction{tx1, tx2, tx3}
block := types.NewBlock(&types.Header{Root: types.EmptyRootHash, Number: big.NewInt(314)}, txs, nil, nil)
block := types.NewBlock(&types.Header{Root: types.EmptyRootHash, Number: big.NewInt(314)}, txs, nil, nil, newHasher())
// Check that no transactions entries are in a pristine database
for i, tx := range txs {

17
core/rawdb/accessors_metadata.go
View file

@ -83,20 +83,3 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha
}
return db.Put(configKey(hash), data)
}
// ReadPreimage returns a Database instance with the key prefix for preimage entries.
func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(preimageKey(hash))
return data
}
// WritePreimages writes the provided set of preimages to the database.
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
for hash, preimage := range preimages {
if err := db.Put(preimageKey(hash), preimage); err != nil {
log.Crit("Failed to store trie preimage", "err", err)
}
}
preimageCounter.Inc(int64(len(preimages)))
preimageHitCounter.Inc(int64(len(preimages)))
}

96
core/rawdb/accessors_state.go Normal file
View file

@ -0,0 +1,96 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rawdb
import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
)
// ReadPreimage retrieves a single preimage of the provided hash.
func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(preimageKey(hash))
return data
}
// WritePreimages writes the provided set of preimages to the database.
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
for hash, preimage := range preimages {
if err := db.Put(preimageKey(hash), preimage); err != nil {
log.Crit("Failed to store trie preimage", "err", err)
}
}
preimageCounter.Inc(int64(len(preimages)))
preimageHitCounter.Inc(int64(len(preimages)))
}
// ReadCode retrieves the contract code of the provided code hash.
func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
// Try with the legacy code scheme first, if not then try with current
// scheme. Since most of the code will be found with legacy scheme.
//
// todo(rjl493456442) change the order when we forcibly upgrade the code
// scheme with snapshot.
data, _ := db.Get(hash[:])
if len(data) != 0 {
return data
}
return ReadCodeWithPrefix(db, hash)
}
// ReadCodeWithPrefix retrieves the contract code of the provided code hash.
// The main difference between this function and ReadCode is this function
// will only check the existence with latest scheme(with prefix).
func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(codeKey(hash))
return data
}
// WriteCode writes the provided contract code database.
func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
if err := db.Put(codeKey(hash), code); err != nil {
log.Crit("Failed to store contract code", "err", err)
}
}
// DeleteCode deletes the specified contract code from the database.
func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(codeKey(hash)); err != nil {
log.Crit("Failed to delete contract code", "err", err)
}
}
// ReadTrieNode retrieves the trie node of the provided hash.
func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(hash.Bytes())
return data
}
// WriteTrieNode writes the provided trie node database.
func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
if err := db.Put(hash.Bytes(), node); err != nil {
log.Crit("Failed to store trie node", "err", err)
}
}
// DeleteTrieNode deletes the specified trie node from the database.
func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(hash.Bytes()); err != nil {
log.Crit("Failed to delete trie node", "err", err)
}
}

4
core/rawdb/database.go
View file

@ -143,6 +143,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
numHashPairing common.StorageSize
hashNumPairing common.StorageSize
trieSize common.StorageSize
codeSize common.StorageSize
txlookupSize common.StorageSize
preimageSize common.StorageSize
bloomBitsSize common.StorageSize
@ -195,6 +196,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
chtTrieNodes += size
case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
bloomTrieNodes += size
case bytes.HasPrefix(key, codePrefix) && len(key) == len(codePrefix)+common.HashLength:
codeSize += size
case len(key) == common.HashLength:
trieSize += size
default:
@ -234,6 +237,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Block hash->number", hashNumPairing.String()},
{"Key-Value store", "Transaction index", txlookupSize.String()},
{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
{"Key-Value store", "Contract codes", codeSize.String()},
{"Key-Value store", "Trie nodes", trieSize.String()},
{"Key-Value store", "Trie preimages", preimageSize.String()},
{"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()},

16
core/rawdb/schema.go
View file

@ -18,6 +18,7 @@
package rawdb
import (
"bytes"
"encoding/binary"
"github.com/XinFinOrg/XDPoSChain/common"
@ -52,6 +53,7 @@ var (
txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
codePrefix = []byte("c") // codePrefix + code hash -> account code
// used by old db, now only used for conversion
oldReceiptsPrefix = []byte("receipts-")
@ -156,6 +158,20 @@ func preimageKey(hash common.Hash) []byte {
return append(preimagePrefix, hash.Bytes()...)
}
// codeKey = codePrefix + hash
func codeKey(hash common.Hash) []byte {
return append(codePrefix, hash.Bytes()...)
}
// IsCodeKey reports whether the given byte slice is the key of contract code,
// if so return the raw code hash as well.
func IsCodeKey(key []byte) (bool, []byte) {
if bytes.HasPrefix(key, codePrefix) && len(key) == common.HashLength+len(codePrefix) {
return true, key[len(codePrefix):]
}
return false, nil
}
// configKey = configPrefix + hash
func configKey(hash common.Hash) []byte {
return append(configPrefix, hash.Bytes()...)

36
core/state/database.go
View file

@ -17,10 +17,12 @@
package state
import (
"errors"
"fmt"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/lru"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/trie"
)
@ -28,6 +30,9 @@ import (
const (
// Number of codehash->size associations to keep.
codeSizeCacheSize = 100000
// Cache size granted for caching clean code.
codeCacheSize = 64 * 1024 * 1024
)
// Database wraps access to tries and contract code.
@ -109,12 +114,14 @@ func NewDatabase(db ethdb.Database) Database {
func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
return &cachingDB{
db: trie.NewDatabaseWithCache(db, cache),
codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
}
}
type cachingDB struct {
db *trie.Database
codeCache *lru.SizeConstrainedCache[common.Hash, []byte]
codeSizeCache *lru.Cache[common.Hash, int]
}
@ -140,11 +147,32 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
// ContractCode retrieves a particular contract's code.
func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) {
code, err := db.db.Node(codeHash)
if err == nil {
db.codeSizeCache.Add(codeHash, len(code))
if code, _ := db.codeCache.Get(codeHash); len(code) > 0 {
return code, nil
}
return code, err
code := rawdb.ReadCode(db.db.DiskDB(), codeHash)
if len(code) > 0 {
db.codeCache.Add(codeHash, code)
db.codeSizeCache.Add(codeHash, len(code))
return code, nil
}
return nil, errors.New("not found")
}
// ContractCodeWithPrefix retrieves a particular contract's code. If the
// code can't be found in the cache, then check the existence with **new**
// db scheme.
func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) {
if code, _ := db.codeCache.Get(codeHash); len(code) > 0 {
return code, nil
}
code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash)
if len(code) > 0 {
db.codeCache.Add(codeHash, code)
db.codeSizeCache.Add(codeHash, len(code))
return code, nil
}
return nil, errors.New("not found")
}
// ContractCodeSize retrieves a particular contracts code's size.

5
core/state/iterator_test.go
View file

@ -42,7 +42,10 @@ func TestNodeIteratorCoverage(t *testing.T) {
}
// Cross check the iterated hashes and the database/nodepool content
for hash := range hashes {
if _, err := db.TrieDB().Node(hash); err != nil {
if _, err = db.TrieDB().Node(hash); err != nil {
_, err = db.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Errorf("failed to retrieve reported node %x", hash)
}
}

32
core/state/statedb.go
View file

@ -25,6 +25,7 @@ import (
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/log"
@ -562,7 +563,10 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject)
newobj.created = true
s.setStateObject(newobj)
return newobj, prev
if prev != nil && !prev.deleted {
return newobj, prev
}
return newobj, nil
}
// CreateAccount explicitly creates a state object. If a state object with the address
@ -755,28 +759,34 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
defer s.clearJournalAndRefund()
// Commit objects to the trie, measuring the elapsed time
for addr, stateObject := range s.stateObjects {
codeWriter := s.db.TrieDB().DiskDB().NewBatch()
for addr, obj := range s.stateObjects {
_, isDirty := s.stateObjectsDirty[addr]
switch {
case stateObject.selfDestructed || (isDirty && deleteEmptyObjects && stateObject.empty()):
case obj.selfDestructed || (isDirty && deleteEmptyObjects && obj.empty()):
// If the object has been removed, don't bother syncing it
// and just mark it for deletion in the trie.
s.deleteStateObject(stateObject)
s.deleteStateObject(obj)
case isDirty:
// Write any contract code associated with the state object
if stateObject.code != nil && stateObject.dirtyCode {
s.db.TrieDB().InsertBlob(common.BytesToHash(stateObject.CodeHash()), stateObject.code)
stateObject.dirtyCode = false
if obj.code != nil && obj.dirtyCode {
rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code)
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie.
if err := stateObject.CommitTrie(s.db); err != nil {
if err := obj.CommitTrie(s.db); err != nil {
return common.Hash{}, err
}
// Update the object in the main account trie.
s.updateStateObject(stateObject)
s.updateStateObject(obj)
}
delete(s.stateObjectsDirty, addr)
}
if codeWriter.ValueSize() > 0 {
if err := codeWriter.Write(); err != nil {
log.Crit("Failed to commit dirty codes", "error", err)
}
}
// Write the account trie changes, measuing the amount of wasted time
defer func(start time.Time) { s.AccountCommits += time.Since(start) }(time.Now())
@ -788,10 +798,6 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error)
if account.Root != types.EmptyRootHash {
s.db.TrieDB().Reference(account.Root, parent)
}
code := common.BytesToHash(account.CodeHash)
if code != types.EmptyCodeHash {
s.db.TrieDB().Reference(code, parent)
}
return nil
})
return root, err

2
core/state/sync.go
View file

@ -34,7 +34,7 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.S
return err
}
syncer.AddSubTrie(obj.Root, 64, parent, nil)
syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent)
syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), 64, parent)
return nil
}
syncer = trie.NewSync(root, database, callback, bloom)

95
core/state/sync_test.go
View file

@ -133,13 +133,17 @@ func TestEmptyStateSync(t *testing.T) {
// Tests that given a root hash, a state can sync iteratively on a single thread,
// requesting retrieval tasks and returning all of them in one go.
func TestIterativeStateSyncIndividual(t *testing.T) { testIterativeStateSync(t, 1) }
func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, 100) }
func TestIterativeStateSyncIndividual(t *testing.T) { testIterativeStateSync(t, 1, false) }
func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, 100, false) }
func TestIterativeStateSyncIndividualFromDisk(t *testing.T) { testIterativeStateSync(t, 1, true) }
func TestIterativeStateSyncBatchedFromDisk(t *testing.T) { testIterativeStateSync(t, 100, true) }
func testIterativeStateSync(t *testing.T, count int) {
func testIterativeStateSync(t *testing.T, count int, commit bool) {
// Create a random state to copy
srcDb, srcRoot, srcAccounts := makeTestState()
if commit {
srcDb.TrieDB().Commit(srcRoot, false)
}
// Create a destination state and sync with the scheduler
dstDb := rawdb.NewMemoryDatabase()
sched := NewStateSync(srcRoot, dstDb, trie.NewSyncBloom(1, dstDb))
@ -149,13 +153,18 @@ func testIterativeStateSync(t *testing.T, count int) {
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
data, err = srcDb.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -184,13 +193,18 @@ func TestIterativeDelayedStateSync(t *testing.T) {
results := make([]trie.SyncResult, len(queue)/2+1)
for i, hash := range queue[:len(results)] {
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
data, err = srcDb.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -226,14 +240,19 @@ func testIterativeRandomStateSync(t *testing.T, count int) {
results := make([]trie.SyncResult, 0, len(queue))
for hash := range queue {
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
data, err = srcDb.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
results = append(results, trie.SyncResult{Hash: hash, Data: data})
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -270,6 +289,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
delete(queue, hash)
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
data, err = srcDb.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
@ -280,8 +302,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) {
}
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -302,6 +326,15 @@ func TestIncompleteStateSync(t *testing.T) {
// Create a random state to copy
srcDb, srcRoot, srcAccounts := makeTestState()
// isCode reports whether the hash is contract code hash.
isCode := func(hash common.Hash) bool {
for _, acc := range srcAccounts {
if hash == crypto.Keccak256Hash(acc.code) {
return true
}
}
return false
}
checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot)
// Create a destination state and sync with the scheduler
@ -315,14 +348,19 @@ func TestIncompleteStateSync(t *testing.T) {
results := make([]trie.SyncResult, len(queue))
for i, hash := range queue {
data, err := srcDb.TrieDB().Node(hash)
if err != nil {
data, err = srcDb.ContractCode(common.Hash{}, hash)
}
if err != nil {
t.Fatalf("failed to retrieve node data for %x", hash)
}
results[i] = trie.SyncResult{Hash: hash, Data: data}
}
// Process each of the state nodes
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := dstDb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -333,12 +371,9 @@ func TestIncompleteStateSync(t *testing.T) {
added = append(added, result.Hash)
}
// Check that all known sub-tries added so far are complete or missing entirely.
checkSubtries:
for _, hash := range added {
for _, acc := range srcAccounts {
if hash == crypto.Keccak256Hash(acc.code) {
continue checkSubtries // skip trie check of code nodes.
}
if isCode(hash) {
continue
}
// Can't use checkStateConsistency here because subtrie keys may have odd
// length and crash in LeafKey.
@ -351,13 +386,25 @@ func TestIncompleteStateSync(t *testing.T) {
}
// Sanity check that removing any node from the database is detected
for _, node := range added[1:] {
key := node.Bytes()
value, _ := dstDb.Get(key)
dstDb.Delete(key)
var (
key = node.Bytes()
code = isCode(node)
val []byte
)
if code {
val = rawdb.ReadCode(dstDb, node)
rawdb.DeleteCode(dstDb, node)
} else {
val = rawdb.ReadTrieNode(dstDb, node)
rawdb.DeleteTrieNode(dstDb, node)
}
if err := checkStateConsistency(dstDb, added[0]); err == nil {
t.Fatalf("trie inconsistency not caught, missing: %x", key)
}
dstDb.Put(key, value)
if code {
rawdb.WriteCode(dstDb, node, val)
} else {
rawdb.WriteTrieNode(dstDb, node, val)
}
}
}

3
core/state_processor_test.go
View file

@ -28,6 +28,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/vm"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
"golang.org/x/crypto/sha3"
)
@ -286,5 +287,5 @@ func GenerateBadBlock(t *testing.T, parent *types.Block, engine consensus.Engine
}
header.Root = common.BytesToHash(hasher.Sum(nil))
// Assemble and return the final block for sealing
return types.NewBlock(header, txs, nil, receipts)
return types.NewBlock(header, txs, nil, receipts, new(trie.Trie))
}

3
core/txpool/txpool_test.go
View file

@ -35,6 +35,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/event"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
)
var (
@ -82,7 +83,7 @@ func (bc *testBlockChain) CurrentBlock() *types.Block {
return types.NewBlock(&types.Header{
Root: types.EmptyRootHash,
GasLimit: atomic.LoadUint64(&bc.gasLimit),
}, nil, nil, nil)
}, nil, nil, nil, new(trie.Trie))
}
func (bc *testBlockChain) GetBlock(hash common.Hash, number uint64) *types.Block {

6
core/types/block.go
View file

@ -216,14 +216,14 @@ type storageblock struct {
// The values of TxHash, UncleHash, ReceiptHash and Bloom in header
// are ignored and set to values derived from the given txs, uncles
// and receipts.
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt) *Block {
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher Hasher) *Block {
b := &Block{header: CopyHeader(header), td: new(big.Int)}
// TODO: panic if len(txs) != len(receipts)
if len(txs) == 0 {
b.header.TxHash = EmptyTxsHash
} else {
b.header.TxHash = DeriveSha(Transactions(txs))
b.header.TxHash = DeriveSha(Transactions(txs), hasher)
b.transactions = make(Transactions, len(txs))
copy(b.transactions, txs)
}
@ -231,7 +231,7 @@ func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*
if len(receipts) == 0 {
b.header.ReceiptHash = EmptyReceiptsHash
} else {
b.header.ReceiptHash = DeriveSha(Receipts(receipts))
b.header.ReceiptHash = DeriveSha(Receipts(receipts), hasher)
b.header.Bloom = CreateBloom(receipts)
}

2
core/types/block_test.go
View file

@ -211,5 +211,5 @@ func makeBenchBlock() *Block {
Extra: []byte("benchmark uncle"),
}
}
return NewBlock(header, txs, uncles, receipts)
return NewBlock(header, txs, uncles, receipts, newHasher())
}

17
core/types/derive_sha.go
View file

@ -21,21 +21,28 @@ import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/trie"
)
// DerivableList is the interface which can derive the hash.
type DerivableList interface {
Len() int
GetRlp(i int) []byte
}
func DeriveSha(list DerivableList) common.Hash {
// Hasher is the tool used to calculate the hash of derivable list.
type Hasher interface {
Reset()
Update([]byte, []byte)
Hash() common.Hash
}
func DeriveSha(list DerivableList, hasher Hasher) common.Hash {
hasher.Reset()
keybuf := new(bytes.Buffer)
trie := new(trie.Trie)
for i := 0; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
trie.Update(keybuf.Bytes(), list.GetRlp(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
}
return trie.Hash()
return hasher.Hash()
}

5
eth/downloader/queue.go
View file

@ -30,6 +30,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
"github.com/XinFinOrg/XDPoSChain/trie"
)
var (
@ -772,7 +773,7 @@ func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLi
defer q.lock.Unlock()
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Transactions(txLists[index])) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
if types.DeriveSha(types.Transactions(txLists[index]), new(trie.Trie)) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
return errInvalidBody
}
result.Transactions = txLists[index]
@ -790,7 +791,7 @@ func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int,
defer q.lock.Unlock()
reconstruct := func(header *types.Header, index int, result *fetchResult) error {
if types.DeriveSha(types.Receipts(receiptList[index])) != header.ReceiptHash {
if types.DeriveSha(types.Receipts(receiptList[index]), new(trie.Trie)) != header.ReceiptHash {
return errInvalidReceipt
}
result.Receipts = receiptList[index]

10
eth/downloader/statesync.go
View file

@ -467,14 +467,12 @@ func (s *stateSync) process(req *stateReq) (int, error) {
}(time.Now())
// Iterate over all the delivered data and inject one-by-one into the trie
progress := false
for _, blob := range req.response {
prog, hash, err := s.processNodeData(blob)
hash, err := s.processNodeData(blob)
switch err {
case nil:
s.numUncommitted++
s.bytesUncommitted += len(blob)
progress = progress || prog
successful++
case trie.ErrNotRequested:
unexpected++
@ -508,13 +506,13 @@ func (s *stateSync) process(req *stateReq) (int, error) {
// processNodeData tries to inject a trie node data blob delivered from a remote
// peer into the state trie, returning whether anything useful was written or any
// error occurred.
func (s *stateSync) processNodeData(blob []byte) (bool, common.Hash, error) {
func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) {
res := trie.SyncResult{Data: blob}
s.keccak.Reset()
s.keccak.Write(blob)
s.keccak.Sum(res.Hash[:0])
committed, _, err := s.sched.Process([]trie.SyncResult{res})
return committed, res.Hash, err
err := s.sched.Process(res)
return res.Hash, err
}
// updateStats bumps the various state sync progress counters and displays a log

5
eth/fetcher/fetcher.go
View file

@ -28,6 +28,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/consensus"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/trie"
)
const (
@ -472,7 +473,7 @@ func (f *Fetcher) loop() {
announce.time = task.time
// If the block is empty (header only), short circuit into the final import queue
if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
if header.TxHash == types.DeriveSha(types.Transactions{}, new(trie.Trie)) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash())
block := types.NewBlockWithHeader(header)
@ -534,7 +535,7 @@ func (f *Fetcher) loop() {
for hash, announce := range f.completing {
if f.queued[hash] == nil {
txnHash := types.DeriveSha(types.Transactions(task.transactions[i]))
txnHash := types.DeriveSha(types.Transactions(task.transactions[i]), new(trie.Trie))
uncleHash := types.CalcUncleHash(task.uncles[i])
if txnHash == announce.header.TxHash && uncleHash == announce.header.UncleHash && announce.origin == task.peer {

3
eth/fetcher/fetcher_test.go
View file

@ -25,6 +25,7 @@ import (
"time"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/trie"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/consensus/ethash"
@ -39,7 +40,7 @@ var (
testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000))
unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit}, nil, nil, nil)
unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit}, nil, nil, nil, new(trie.Trie))
)
// makeChain creates a chain of n blocks starting at and including parent.

9
eth/handler.go
View file

@ -633,7 +633,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
// Retrieve the requested state entry, stopping if enough was found
if entry, err := pm.blockchain.TrieNode(hash); err == nil {
// todo now the code and trienode is mixed in the protocol level,
// separate these two types.
entry, err := pm.blockchain.TrieNode(hash)
if len(entry) == 0 || err != nil {
// Read the contract code with prefix only to save unnecessary lookups.
entry, err = pm.blockchain.ContractCodeWithPrefix(hash)
}
if err == nil && len(entry) > 0 {
data = append(data, entry)
bytes += len(entry)
}

4
internal/ethapi/trie_proof_test.go
View file

@ -49,7 +49,7 @@ func TestTransactionProof(t *testing.T) {
transactions := types.Transactions([]*types.Transaction{t1, t2, t3, t4})
tr := deriveTrie(transactions)
// for verifying the proof
root := types.DeriveSha(transactions)
root := types.DeriveSha(transactions, new(trie.Trie))
for i := 0; i < transactions.Len(); i++ {
var proof proofPairList
keybuf := new(bytes.Buffer)
@ -82,7 +82,7 @@ func TestReceiptProof(t *testing.T) {
receipts := types.Receipts([]*types.Receipt{r1, r2, r3, r4})
tr := deriveTrie(receipts)
// for verifying the proof
root := types.DeriveSha(receipts)
root := types.DeriveSha(receipts, new(trie.Trie))
for i := 0; i < receipts.Len(); i++ {
var proof proofPairList
keybuf := new(bytes.Buffer)

2
miner/worker.go
View file

@ -43,6 +43,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/event"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/params"
"github.com/XinFinOrg/XDPoSChain/trie"
mapset "github.com/deckarep/golang-set/v2"
)
@ -536,6 +537,7 @@ func (w *worker) updateSnapshot() {
w.current.txs,
nil,
w.current.receipts,
new(trie.Trie),
)
w.snapshotReceipts = copyReceipts(w.current.receipts)
w.snapshotState = w.current.state.Copy()

129
trie/database.go
View file

@ -26,6 +26,7 @@ import (
"github.com/VictoriaMetrics/fastcache"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
@ -56,15 +57,6 @@ var (
memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil)
)
// secureKeyPrefix is the database key prefix used to store trie Node preimages.
var secureKeyPrefix = []byte("secure-key-")
// secureKeyPrefixLength is the length of the above prefix
const secureKeyPrefixLength = 11
// secureKeyLength is the length of the above prefix + 32byte hash.
const secureKeyLength = secureKeyPrefixLength + 32
// Database is an intermediate write layer between the trie data structures and
// the disk database. The aim is to accumulate trie writes in-memory and only
// periodically flush a couple tries to disk, garbage collecting the remainder.
@ -77,7 +69,7 @@ type Database struct {
diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes
cleans *fastcache.Cache // GC friendly memory Cache of clean Node RLPs
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty trie nodes
oldest common.Hash // Oldest tracked Node, flush-list head
newest common.Hash // Newest tracked Node, flush-list tail
@ -138,8 +130,8 @@ type rawShortNode struct {
func (n rawShortNode) Cache() (HashNode, bool) { panic("this should never end up in a live trie") }
func (n rawShortNode) fstring(ind string) string { panic("this should never end up in a live trie") }
// cachedNode is all the information we know about a single cached Node in the
// memory database write layer.
// cachedNode is all the information we know about a single cached trie node
// in the memory database write layer.
type cachedNode struct {
node Node // Cached collapsed trie Node, or raw rlp data
size uint16 // Byte size of the useful cached data
@ -160,8 +152,8 @@ var cachedNodeSize = int(reflect.TypeOf(cachedNode{}).Size())
// reference map.
const cachedNodeChildrenSize = 48
// rlp returns the raw rlp encoded blob of the cached Node, either directly from
// the Cache, or by regenerating it from the collapsed Node.
// rlp returns the raw rlp encoded blob of the cached trie node, either directly
// from the cache, or by regenerating it from the collapsed node.
func (n *cachedNode) rlp() []byte {
if node, ok := n.node.(rawNode); ok {
return node
@ -182,9 +174,9 @@ func (n *cachedNode) obj(hash common.Hash) Node {
return expandNode(hash[:], n.node)
}
// forChilds invokes the callback for all the tracked children of this Node,
// both the implicit ones from inside the Node as well as the explicit ones
//from outside the Node.
// forChilds invokes the callback for all the tracked children of this node,
// both the implicit ones from inside the node as well as the explicit ones
// from outside the node.
func (n *cachedNode) forChilds(onChild func(hash common.Hash)) {
for child := range n.children {
onChild(child)
@ -300,25 +292,14 @@ func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database {
}
// DiskDB retrieves the persistent storage backing the trie database.
func (db *Database) DiskDB() ethdb.KeyValueReader {
func (db *Database) DiskDB() ethdb.KeyValueStore {
return db.diskdb
}
// InsertBlob writes a new reference tracked blob to the memory database if it's
// yet unknown. This method should only be used for non-trie nodes that require
// reference counting, since trie nodes are garbage collected directly through
// their embedded children.
func (db *Database) InsertBlob(hash common.Hash, blob []byte) {
db.Lock.Lock()
defer db.Lock.Unlock()
db.insert(hash, len(blob), rawNode(blob))
}
// insert inserts a collapsed trie Node into the memory database. This method is
// a more generic version of InsertBlob, supporting both raw blob insertions as
// well ex trie Node insertions. The blob size must be specified to allow proper
// size tracking.
// insert inserts a collapsed trie node into the memory database.
// The blob size must be specified to allow proper size tracking.
// All nodes inserted by this function will be reference tracked
// and in theory should only used for **trie nodes** insertion.
func (db *Database) insert(hash common.Hash, size int, node Node) {
// If the Node's already cached, skip
if _, ok := db.dirties[hash]; ok {
@ -424,39 +405,30 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
memcacheDirtyMissMeter.Mark(1)
// Content unavailable in memory, attempt to retrieve from disk
enc, err := db.diskdb.Get(hash[:])
if err == nil && enc != nil {
enc := rawdb.ReadTrieNode(db.diskdb, hash)
if len(enc) != 0 {
if db.cleans != nil {
db.cleans.Set(hash[:], enc)
memcacheCleanMissMeter.Mark(1)
memcacheCleanWriteMeter.Mark(int64(len(enc)))
}
return enc, nil
}
return enc, err
return nil, errors.New("not found")
}
// Preimage retrieves a cached trie Node pre-image from memory. If it cannot be
// found cached, the method queries the persistent database for the content.
func (db *Database) Preimage(hash common.Hash) ([]byte, error) {
func (db *Database) Preimage(hash common.Hash) []byte {
// Retrieve the Node from Cache if available
db.Lock.RLock()
preimage := db.preimages[hash]
db.Lock.RUnlock()
if preimage != nil {
return preimage, nil
return preimage
}
// Content unavailable in memory, attempt to retrieve from disk
return db.diskdb.Get(secureKey(hash))
}
// secureKey returns the database key for the Preimage of key (as a newly
// allocated byte-slice)
func secureKey(hash common.Hash) []byte {
buf := make([]byte, secureKeyLength)
copy(buf, secureKeyPrefix)
copy(buf[secureKeyPrefixLength:], hash[:])
return buf
return rawdb.ReadPreimage(db.diskdb, hash)
}
// Nodes retrieves the hashes of all the nodes cached within the memory database.
@ -475,7 +447,10 @@ func (db *Database) Nodes() []common.Hash {
return hashes
}
// Reference adds a new reference from a parent Node to a child Node.
// Reference adds a new reference from a parent node to a child node.
// This function is used to add reference between internal trie node
// and external node(e.g. storage trie root), all internal trie nodes
// are referenced together by database itself.
func (db *Database) Reference(child common.Hash, parent common.Hash) {
db.Lock.Lock()
defer db.Lock.Unlock()
@ -598,27 +573,16 @@ func (db *Database) Cap(limit common.StorageSize) error {
size := db.dirtiesSize + common.StorageSize((len(db.dirties)-1)*cachedNodeSize)
size += db.childrenSize - common.StorageSize(len(db.dirties[common.Hash{}].children)*(common.HashLength+2))
// We reuse an ephemeral buffer for the keys. The batch Put operation
// copies it internally, so we can reuse it.
var keyBuf [secureKeyLength]byte
copy(keyBuf[:], secureKeyPrefix)
// If the Preimage Cache got large enough, push to disk. If it's still small
// leave for later to deduplicate writes.
flushPreimages := db.preimagesSize > 4*1024*1024
if flushPreimages {
for hash, preimage := range db.preimages {
copy(keyBuf[secureKeyPrefixLength:], hash[:])
if err := batch.Put(keyBuf[:], preimage); err != nil {
log.Error("Failed to commit Preimage from trie database", "err", err)
rawdb.WritePreimages(batch, db.preimages)
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err
}
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err
}
batch.Reset()
}
batch.Reset()
}
}
// Keep committing nodes from the flush-list until we're below allowance
@ -626,9 +590,8 @@ func (db *Database) Cap(limit common.StorageSize) error {
for size > limit && oldest != (common.Hash{}) {
// Fetch the oldest referenced Node and push into the batch
node := db.dirties[oldest]
if err := batch.Put(oldest[:], node.rlp()); err != nil {
return err
}
rawdb.WriteTrieNode(batch, oldest, node.rlp())
// If we exceeded the ideal batch size, commit and reset
if batch.ValueSize() >= ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
@ -656,8 +619,7 @@ func (db *Database) Cap(limit common.StorageSize) error {
defer db.Lock.Unlock()
if flushPreimages {
db.preimages = make(map[common.Hash][]byte)
db.preimagesSize = 0
db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0
}
for db.oldest != oldest {
node := db.dirties[db.oldest]
@ -700,25 +662,13 @@ func (db *Database) Commit(node common.Hash, report bool) error {
start := time.Now()
batch := db.diskdb.NewBatch()
// We reuse an ephemeral buffer for the keys. The batch Put operation
// copies it internally, so we can reuse it.
var keyBuf [secureKeyLength]byte
copy(keyBuf[:], secureKeyPrefix)
// Move all of the accumulated preimages into a write batch
for hash, preimage := range db.preimages {
copy(keyBuf[secureKeyPrefixLength:], hash[:])
if err := batch.Put(keyBuf[:], preimage); err != nil {
log.Error("Failed to commit Preimage from trie database", "err", err)
rawdb.WritePreimages(batch, db.preimages)
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err
}
// If the batch is too large, flush to disk
if batch.ValueSize() > ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err
}
batch.Reset()
}
batch.Reset()
}
// Since we're going to replay trie Node writes into the clean Cache, flush out
// any batched pre-images before continuing.
@ -748,8 +698,7 @@ func (db *Database) Commit(node common.Hash, report bool) error {
batch.Reset()
// Reset the storage counters and bumpd metrics
db.preimages = make(map[common.Hash][]byte)
db.preimagesSize = 0
db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0
memcacheCommitTimeTimer.Update(time.Since(start))
memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize))
@ -785,10 +734,8 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane
if err != nil {
return err
}
if err := batch.Put(hash[:], node.rlp()); err != nil {
return err
}
// If we've reached an optimal batch size, commit and start over
rawdb.WriteTrieNode(batch, hash, node.rlp())
if batch.ValueSize() >= ethdb.IdealBatchSize {
if err := batch.Write(); err != nil {
return err

3
trie/secure_trie.go
View file

@ -130,8 +130,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte {
if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
return key
}
key, _ := t.trie.Db.Preimage(common.BytesToHash(shaKey))
return key
return t.trie.Db.Preimage(common.BytesToHash(shaKey))
}
// Commit writes all nodes and the secure hash pre-images to the trie's database.

193
trie/sync.go
View file

@ -22,6 +22,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/prque"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/ethdb"
)
@ -37,7 +38,7 @@ var ErrAlreadyProcessed = errors.New("already processed")
type request struct {
hash common.Hash // Hash of the Node data content to retrieve
data []byte // Data content of the Node, cached until all subtrees complete
raw bool // Whether this is a raw entry (code) or a trie Node
code bool // Whether this is a code entry
parents []*request // Parent state nodes referencing this entry (notify all upon completion)
depth int // Depth level within the trie the Node is located to prioritise DFS
@ -46,8 +47,7 @@ type request struct {
callback LeafCallback // Callback to invoke if a leaf Node it reached on this branch
}
// SyncResult is a simple list to return missing nodes along with their request
// hashes.
// SyncResult is a response with requested data along with it's hash.
type SyncResult struct {
Hash common.Hash // Hash of the originally unknown trie Node
Data []byte // Data content of the retrieved Node
@ -56,25 +56,40 @@ type SyncResult struct {
// syncMemBatch is an in-memory buffer of successfully downloaded but not yet
// persisted data items.
type syncMemBatch struct {
batch map[common.Hash][]byte // In-memory membatch of recently completed items
nodes map[common.Hash][]byte // In-memory membatch of recently completed nodes
codes map[common.Hash][]byte // In-memory membatch of recently completed codes
}
// newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes.
func newSyncMemBatch() *syncMemBatch {
return &syncMemBatch{
batch: make(map[common.Hash][]byte),
nodes: make(map[common.Hash][]byte),
codes: make(map[common.Hash][]byte),
}
}
// hasNode reports the trie node with specific hash is already cached.
func (batch *syncMemBatch) hasNode(hash common.Hash) bool {
_, ok := batch.nodes[hash]
return ok
}
// hasCode reports the contract code with specific hash is already cached.
func (batch *syncMemBatch) hasCode(hash common.Hash) bool {
_, ok := batch.codes[hash]
return ok
}
// Sync is the main state trie synchronisation scheduler, which provides yet
// unknown trie hashes to retrieve, accepts Node data associated with said hashes
// and reconstructs the trie step by step until all is done.
type Sync struct {
database ethdb.KeyValueReader // Persistent database to check for existing entries
membatch *syncMemBatch // Memory buffer to avoid frequent database writes
requests map[common.Hash]*request // Pending requests pertaining to a key hash
nodeReqs map[common.Hash]*request // Pending requests pertaining to a trie node hash
codeReqs map[common.Hash]*request // Pending requests pertaining to a code hash
queue *prque.Prque[int64, any] // Priority queue with the pending requests
bloom *SyncBloom // Bloom filter for fast Node existence checks
bloom *SyncBloom // Bloom filter for fast state existence checks
}
// NewSync creates a new trie data download scheduler.
@ -82,7 +97,8 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb
ts := &Sync{
database: database,
membatch: newSyncMemBatch(),
requests: make(map[common.Hash]*request),
nodeReqs: make(map[common.Hash]*request),
codeReqs: make(map[common.Hash]*request),
queue: prque.New[int64, any](nil), // Ugh, can contain both string and hash, whyyy
bloom: bloom,
}
@ -96,13 +112,15 @@ func (s *Sync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callb
if root == emptyRoot {
return
}
if _, ok := s.membatch.batch[root]; ok {
if s.membatch.hasNode(root) {
return
}
if s.bloom.Contains(root[:]) {
// Bloom filter says this might be a duplicate, double check
blob, _ := s.database.Get(root[:])
if local, err := decodeNode(root[:], blob); local != nil && err == nil {
if s.bloom == nil || s.bloom.Contains(root[:]) {
// Bloom filter says this might be a duplicate, double check.
// If database says yes, then at least the trie node is present
// and we hold the assumption that it's NOT legacy contract code.
blob := rawdb.ReadTrieNode(s.database, root)
if len(blob) > 0 {
return
}
// False positive, bump fault meter
@ -116,7 +134,7 @@ func (s *Sync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callb
}
// If this sub-trie has a designated parent, link them together
if parent != (common.Hash{}) {
ancestor := s.requests[parent]
ancestor := s.nodeReqs[parent]
if ancestor == nil {
panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
}
@ -126,21 +144,25 @@ func (s *Sync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callb
s.schedule(req)
}
// AddRawEntry schedules the direct retrieval of a state entry that should not be
// interpreted as a trie Node, but rather accepted and stored into the database
// as is. This method's goal is to support misc state metadata retrievals (e.g.
// contract code).
func (s *Sync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
// AddCodeEntry schedules the direct retrieval of a contract code that should not
// be interpreted as a trie node, but rather accepted and stored into the database
// as is.
func (s *Sync) AddCodeEntry(hash common.Hash, depth int, parent common.Hash) {
// Short circuit if the entry is empty or already known
if hash == emptyState {
return
}
if _, ok := s.membatch.batch[hash]; ok {
if s.membatch.hasCode(hash) {
return
}
if s.bloom.Contains(hash[:]) {
// Bloom filter says this might be a duplicate, double check
if ok, _ := s.database.Has(hash[:]); ok {
// Bloom filter says this might be a duplicate, double check.
// If database says yes, the blob is present for sure.
// Note we only check the existence with new code scheme, fast
// sync is expected to run with a fresh new node. Even there
// exists the code with legacy format, fetch and store with
// new scheme anyway.
if blob := rawdb.ReadCodeWithPrefix(s.database, hash); len(blob) > 0 {
return
}
// False positive, bump fault meter
@ -149,12 +171,12 @@ func (s *Sync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
// Assemble the new sub-trie sync request
req := &request{
hash: hash,
raw: true,
code: true,
depth: depth,
}
// If this sub-trie has a designated parent, link them together
if parent != (common.Hash{}) {
ancestor := s.requests[parent]
ancestor := s.nodeReqs[parent] // the parent of codereq can ONLY be nodereq
if ancestor == nil {
panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
}
@ -173,61 +195,64 @@ func (s *Sync) Missing(max int) []common.Hash {
return requests
}
// Process injects a batch of retrieved trie nodes data, returning if something
// was committed to the database and also the index of an entry if its processing
// failed.
func (s *Sync) Process(results []SyncResult) (bool, int, error) {
committed := false
for i, item := range results {
// If the item was not requested, bail out
request := s.requests[item.Hash]
if request == nil {
return committed, i, ErrNotRequested
}
if request.data != nil {
return committed, i, ErrAlreadyProcessed
}
// If the item is a raw entry request, commit directly
if request.raw {
request.data = item.Data
s.commit(request)
committed = true
continue
}
// Process injects the received data for requested item. Note it can
// happpen that the single response commits two pending requests(e.g.
// there are two requests one for code and one for node but the hash
// is same). In this case the second response for the same hash will
// be treated as "non-requested" item or "already-processed" item but
// there is no downside.
func (s *Sync) Process(result SyncResult) error {
// If the item was not requested either for code or node, bail out
if s.nodeReqs[result.Hash] == nil && s.codeReqs[result.Hash] == nil {
return ErrNotRequested
}
// There is an pending code request for this data, commit directly
var filled bool
if req := s.codeReqs[result.Hash]; req != nil && req.data == nil {
filled = true
req.data = result.Data
s.commit(req)
}
// There is an pending node request for this data, fill it.
if req := s.nodeReqs[result.Hash]; req != nil && req.data == nil {
filled = true
// Decode the Node data content and update the request
node, err := decodeNode(item.Hash[:], item.Data)
node, err := decodeNode(result.Hash[:], result.Data)
if err != nil {
return committed, i, err
return err
}
request.data = item.Data
req.data = result.Data
// Create and schedule a request for all the children nodes
requests, err := s.children(request, node)
requests, err := s.children(req, node)
if err != nil {
return committed, i, err
return err
}
if len(requests) == 0 && request.deps == 0 {
s.commit(request)
committed = true
continue
}
request.deps += len(requests)
for _, child := range requests {
s.schedule(child)
if len(requests) == 0 && req.deps == 0 {
s.commit(req)
} else {
req.deps += len(requests)
for _, child := range requests {
s.schedule(child)
}
}
}
return committed, 0, nil
if !filled {
return ErrAlreadyProcessed
}
return nil
}
// Commit flushes the data stored in the internal membatch out to persistent
// storage, returning any occurred error.
func (s *Sync) Commit(dbw ethdb.Batch) error {
// Dump the membatch into a database dbw
for key, value := range s.membatch.batch {
if err := dbw.Put(key[:], value); err != nil {
return err
}
for key, value := range s.membatch.nodes {
rawdb.WriteTrieNode(dbw, key, value)
s.bloom.Add(key[:])
}
for key, value := range s.membatch.codes {
rawdb.WriteCode(dbw, key, value)
s.bloom.Add(key[:])
}
// Drop the membatch data and return
@ -237,21 +262,30 @@ func (s *Sync) Commit(dbw ethdb.Batch) error {
// Pending returns the number of state entries currently pending for download.
func (s *Sync) Pending() int {
return len(s.requests)
return len(s.nodeReqs) + len(s.codeReqs)
}
// schedule inserts a new state retrieval request into the fetch queue. If there
// is already a pending request for this Node, the new request will be discarded
// and only a parent reference added to the old one.
func (s *Sync) schedule(req *request) {
// If we're already requesting this Node, add a new reference and stop
if old, ok := s.requests[req.hash]; ok {
var reqset = s.nodeReqs
if req.code {
reqset = s.codeReqs
}
// If we're already requesting this node, add a new reference and stop
if old, ok := reqset[req.hash]; ok {
old.parents = append(old.parents, req.parents...)
return
}
// Schedule the request for future retrieval
reqset[req.hash] = req
// Schedule the request for future retrieval. This queue is shared
// by both node requests and code requests. It can happen that there
// is a trie node and code has same hash. In this case two elements
// with same hash and same or different depth will be pushed. But it's
// ok the worst case is the second response will be treated as duplicated.
s.queue.Push(req.hash, int64(req.depth))
s.requests[req.hash] = req
}
// children retrieves all the missing children of a state trie entry for future
@ -297,12 +331,14 @@ func (s *Sync) children(req *request, object Node) ([]*request, error) {
if node, ok := (child.node).(HashNode); ok {
// Try to resolve the Node from the local database
hash := common.BytesToHash(node)
if _, ok := s.membatch.batch[hash]; ok {
if s.membatch.hasNode(hash) {
continue
}
if s.bloom.Contains(node) {
// Bloom filter says this might be a duplicate, double check
if ok, _ := s.database.Has(node); ok {
if s.bloom == nil || s.bloom.Contains(node) {
// Bloom filter says this might be a duplicate, double check.
// If database says yes, then at least the trie node is present
// and we hold the assumption that it's NOT legacy contract code.
if blob := rawdb.ReadTrieNode(s.database, common.BytesToHash(node)); len(blob) > 0 {
continue
}
// False positive, bump fault meter
@ -325,10 +361,13 @@ func (s *Sync) children(req *request, object Node) ([]*request, error) {
// committed themselves.
func (s *Sync) commit(req *request) (err error) {
// Write the Node content to the membatch
s.membatch.batch[req.hash] = req.data
delete(s.requests, req.hash)
if req.code {
s.membatch.codes[req.hash] = req.data
delete(s.codeReqs, req.hash)
} else {
s.membatch.nodes[req.hash] = req.data
delete(s.nodeReqs, req.hash)
}
// Check all parents for completion
for _, parent := range req.parents {
parent.deps--

19
trie/sync_bloom.go
View file

@ -25,6 +25,7 @@ import (
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/metrics"
@ -41,8 +42,8 @@ var (
)
// syncBloomHasher is a wrapper around a byte blob to satisfy the interface API
// requirements of the bloom library used. It's used to convert a trie hash into
// a 64 bit mini hash.
// requirements of the bloom library used. It's used to convert a trie hash or
// contract code hash into a 64 bit mini hash.
type syncBloomHasher []byte
func (f syncBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
@ -53,9 +54,9 @@ func (f syncBloomHasher) Size() int { return 8 }
func (f syncBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) }
// SyncBloom is a bloom filter used during fast sync to quickly decide if a trie
// Node already exists on disk or not. It self populates from the provided disk
// database on creation in a background thread and will only start returning live
// results once that's finished.
// node or contract code already exists on disk or not. It self populates from the
// provided disk database on creation in a background thread and will only start
// returning live results once that's finished.
type SyncBloom struct {
bloom *bloomfilter.Filter
inited uint32
@ -107,10 +108,16 @@ func (b *SyncBloom) init(database ethdb.Iteratee) {
)
for it.Next() && atomic.LoadUint32(&b.closed) == 0 {
// If the database entry is a trie Node, add it to the bloom
if key := it.Key(); len(key) == common.HashLength {
key := it.Key()
if len(key) == common.HashLength {
b.bloom.Add(syncBloomHasher(key))
bloomLoadMeter.Mark(1)
}
// If the database entry is a contract code, add it to the bloom
if ok, hash := rawdb.IsCodeKey(key); ok {
b.bloom.Add(syncBloomHasher(hash))
bloomLoadMeter.Mark(1)
}
// If enough time elapsed since the last iterator swap, restart
if time.Since(swap) > 8*time.Second {
key := common.CopyBytes(it.Key())

36
trie/sync_test.go
View file

@ -124,8 +124,10 @@ func testIterativeSync(t *testing.T, count int) {
}
results[i] = SyncResult{hash, data}
}
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -160,8 +162,10 @@ func TestIterativeDelayedSync(t *testing.T) {
}
results[i] = SyncResult{hash, data}
}
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -204,8 +208,10 @@ func testIterativeRandomSync(t *testing.T, count int) {
results = append(results, SyncResult{hash, data})
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -251,8 +257,10 @@ func TestIterativeRandomDelayedSync(t *testing.T) {
}
}
// Feed the retrieved results back and queue new tasks
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -298,8 +306,10 @@ func TestDuplicateAvoidanceSync(t *testing.T) {
results[i] = SyncResult{hash, data}
}
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {
@ -336,8 +346,10 @@ func TestIncompleteSync(t *testing.T) {
results[i] = SyncResult{hash, data}
}
// Process each of the trie nodes
if _, index, err := sched.Process(results); err != nil {
t.Fatalf("failed to process result #%d: %v", index, err)
for _, result := range results {
if err := sched.Process(result); err != nil {
t.Fatalf("failed to process result %v", err)
}
}
batch := diskdb.NewBatch()
if err := sched.Commit(batch); err != nil {

10
trie/trie.go
View file

@ -579,7 +579,7 @@ func (t *Trie) resolveHash(n HashNode, prefix []byte) (Node, error) {
// Hash returns the root hash of the trie. It does not write to the
// database and can be used even if the trie doesn't have one.
func (t *Trie) Hash() common.Hash {
hash, cached, _ := t.hashRoot(nil)
hash, cached, _ := t.hashRoot()
t.root = cached
return common.BytesToHash(hash.(HashNode))
}
@ -630,7 +630,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) {
}
// hashRoot calculates the root hash of the given trie
func (t *Trie) hashRoot(db *Database) (Node, Node, error) {
func (t *Trie) hashRoot() (Node, Node, error) {
if t.root == nil {
return HashNode(emptyRoot.Bytes()), nil, nil
}
@ -641,3 +641,9 @@ func (t *Trie) hashRoot(db *Database) (Node, Node, error) {
t.unhashed = 0
return hashed, cached, nil
}
// Reset drops the referenced root node and cleans all internal state.
func (t *Trie) Reset() {
t.root = nil
t.unhashed = 0
}