diff --git a/core/blockchain.go b/core/blockchain.go
index d19fe6dd16..d2ba7a8681 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -1679,6 +1679,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []
switch status {
case CanonStatTy:
+ bc.hc.hashHistory.Set(block.Header())
log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
"uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
"elapsed", common.PrettyDuration(time.Since(start)),
@@ -1967,6 +1968,8 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
// Write lookup entries for hash based transaction/receipt searches
rawdb.WriteTxLookupEntries(bc.db, newChain[i])
addedTxs = append(addedTxs, newChain[i].Transactions()...)
+ // Add to hash history
+ bc.hc.hashHistory.Set(newChain[i].Header())
}
// When transactions get deleted from the database, the receipts that were
// created in the fork must also be deleted
@@ -2224,3 +2227,8 @@ func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript
func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription {
return bc.scope.Track(bc.blockProcFeed.Subscribe(ch))
}
+
+//GetAncestorHash return the hash of ancestor at the given number
+func (bc *BlockChain) GetAncestorHash(ref *types.Header, target uint64) common.Hash {
+ return bc.hc.GetAncestorHash(ref, target)
+}
diff --git a/core/evm.go b/core/evm.go
index b654bbd479..75c40e2752 100644
--- a/core/evm.go
+++ b/core/evm.go
@@ -31,8 +31,8 @@ type ChainContext interface {
// Engine retrieves the chain's consensus engine.
Engine() consensus.Engine
- // GetHeader returns the hash corresponding to their hash.
- GetHeader(common.Hash, uint64) *types.Header
+ // GetAncestorHash return the hash of ancestor at the given number
+ GetAncestorHash(child *types.Header, number uint64) common.Hash
}
// NewEVMContext creates a new context for use in the EVM.
@@ -60,27 +60,8 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author
// GetHashFn returns a GetHashFunc which retrieves header hashes by number
func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash {
- var cache map[uint64]common.Hash
-
return func(n uint64) common.Hash {
- // If there's no hash cache yet, make one
- if cache == nil {
- cache = map[uint64]common.Hash{
- ref.Number.Uint64() - 1: ref.ParentHash,
- }
- }
- // Try to fulfill the request from the cache
- if hash, ok := cache[n]; ok {
- return hash
- }
- // Not cached, iterate the blocks and cache the hashes
- for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) {
- cache[header.Number.Uint64()-1] = header.ParentHash
- if n == header.Number.Uint64()-1 {
- return header.ParentHash
- }
- }
- return common.Hash{}
+ return chain.GetAncestorHash(ref, n)
}
}
diff --git a/core/hashbuffer.go b/core/hashbuffer.go
new file mode 100644
index 0000000000..496e5afae1
--- /dev/null
+++ b/core/hashbuffer.go
@@ -0,0 +1,173 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package core
+
+import (
+ "sync"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
+)
+
+// For EVM execution, we need around 256 items. We add a few more to allow reorgs.
+// For mainnet, a couple more would suffice, but a few more added for
+// testnets/private nets
+// For LES, we use a larger buffer. 500K * 32 bytes = 16M
+const hashBufferElems = 500000
+
+var (
+ hashHitCounter = metrics.NewRegisteredGauge("chain/headerhash/hit", nil)
+ hashMissCounter = metrics.NewRegisteredGauge("chain/headerhash/miss", nil)
+ hashHeadGauge = metrics.NewRegisteredGauge("chain/headerhash/head", nil)
+ hashTailGauge = metrics.NewRegisteredGauge("chain/headerhash/tail", nil)
+)
+
+// hashBuffer implements a storage for chains of hashes, intended to be used for quick lookup of block hashes.
+// Internally, it uses an array of hashes in a circular buffer.
+// It enforces that all hashes added have a contiguous parent-child relation, and supports rollbacks
+// It is thread-safe.
+type hashBuffer struct {
+ // The data holds the hashes. The hashes are sequential, but also a
+ // circular buffer.
+ // The `head` points to the position of the latest hash.
+ // The parent, if present, is located 32 bytes back.
+ // [.., .., ..., head-2 , head-1, head, oldest, ... ]
+ data [hashBufferElems]common.Hash
+
+ head uint64 // index of hash for the head block
+
+ headNumber uint64 // The block number for head (the most recent block)
+ tailNumber uint64 // The block number for tail (the oldest block)
+
+ mu sync.RWMutex
+}
+
+// newHashBuffer creates a new storage with a header in it.
+// Since we take a header here, a hash storage can never be empty.
+// This makes things easier later on (in Set)
+func newHashBuffer(header *types.Header) *hashBuffer {
+ return &hashBuffer{
+ headNumber: header.Number.Uint64(),
+ tailNumber: header.Number.Uint64(),
+ data: [hashBufferElems]common.Hash{header.Hash()},
+ }
+}
+
+// Get locates the hash for the requested number
+func (hs *hashBuffer) Get(number uint64) (common.Hash, bool) {
+ hs.mu.RLock()
+ defer hs.mu.RUnlock()
+ if !hs.has(number) {
+ hashMissCounter.Inc(1)
+ return common.Hash{}, false
+ }
+ hashHitCounter.Inc(1)
+ distance := hs.headNumber - number
+ index := (hs.head + hashBufferElems - distance) % hashBufferElems
+ return hs.data[index], true
+}
+
+// has returns if the storage has a hash for the given number
+func (hs *hashBuffer) has(number uint64) bool {
+ return number <= hs.headNumber && number >= hs.tailNumber
+}
+
+// Contains checks if the hash at the given number matches the expected
+func (hs *hashBuffer) Contains(number uint64, expected common.Hash) bool {
+ hs.mu.RLock()
+ defer hs.mu.RUnlock()
+ if hs.contains(number, expected) {
+ hashHitCounter.Inc(1)
+ return true
+ } else {
+ hashMissCounter.Inc(1)
+ return false
+ }
+}
+
+// contains is the non-concurrency safe internal version of Contains
+func (hs *hashBuffer) contains(number uint64, expected common.Hash) bool {
+ if !hs.has(number) {
+ return false
+ }
+ distance := hs.headNumber - number
+ index := (hs.head + hashBufferElems - distance) % hashBufferElems
+ return hs.data[index] == expected
+}
+
+// Newest returns the most recent (number, hash) stored
+func (hs *hashBuffer) Newest() (uint64, common.Hash) {
+ hs.mu.RLock()
+ defer hs.mu.RUnlock()
+ return hs.headNumber, hs.data[hs.head]
+}
+
+// Oldest returns the oldest (number, hash) found
+func (hs *hashBuffer) Oldest() (uint64, common.Hash) {
+ hs.mu.RLock()
+ defer hs.mu.RUnlock()
+ distance := hs.headNumber - hs.tailNumber
+ index := (hs.head + hashBufferElems - distance) % hashBufferElems
+ return hs.tailNumber, hs.data[index]
+}
+
+// Set inserts a new header (hash) to the storage.
+// If
+// a) Header already exists, this is a no-op
+// b) Number is occupied by other header, the new header replaces it, and also
+// truncates any descendants
+//
+// If the new header does not have any ancestors, it replaces the entire storage.
+func (hs *hashBuffer) Set(header *types.Header) {
+ var (
+ number = header.Number.Uint64()
+ index uint64
+ hash = header.Hash()
+ )
+ hs.mu.Lock()
+ defer hs.mu.Unlock()
+ if hs.contains(number-1, header.ParentHash) {
+ if hs.headNumber >= number {
+ distance := hs.headNumber - number
+ index = (hs.head + hashBufferElems - distance) % hashBufferElems
+ if hs.data[index] == hash {
+ return
+ }
+ // Continue by replacing this number and wipe descendants
+ } else {
+ // head is parent of this new header - regular append
+ index = (hs.head + 1) % hashBufferElems
+ }
+ } else {
+ // This should not normally happen, and indicates a programming error
+ log.Error("Hash storage wiping ancestors", "oldhead", hs.headNumber, "newhead", number, "oldtail", hs.tailNumber)
+ // Wipe ancestors
+ hs.tailNumber = number
+ }
+ hs.head = index
+ hs.headNumber = number
+ hs.data[hs.head] = hash
+
+ if number-hs.tailNumber == hashBufferElems {
+ // It's full, need to move the tail
+ hs.tailNumber++
+ }
+ hashTailGauge.Update(int64(hs.tailNumber))
+ hashHeadGauge.Update(int64(hs.headNumber))
+}
diff --git a/core/hashbuffer_test.go b/core/hashbuffer_test.go
new file mode 100644
index 0000000000..960673e963
--- /dev/null
+++ b/core/hashbuffer_test.go
@@ -0,0 +1,204 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package core
+
+import (
+ "bytes"
+ "math/big"
+ "math/rand"
+ "testing"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+func init() {
+ rand.Seed(time.Now().Unix())
+}
+func dummyHeader(n int, prev common.Hash) *types.Header {
+ return &types.Header{
+ Number: new(big.Int).SetUint64(uint64(n)),
+ ParentHash: prev,
+ }
+}
+
+func uniqueHeader(n int, prev common.Hash) *types.Header {
+ return &types.Header{
+ Number: new(big.Int).SetUint64(uint64(n)),
+ ParentHash: prev,
+ GasUsed: rand.Uint64(),
+ }
+}
+
+// TestConsecutiveHashes does inserts and rollback, but uses contiguous chains
+func TestConsecutiveHashes(t *testing.T) {
+ t.Parallel()
+ // This should we swapped out very quickly
+ hs := newHashBuffer(uniqueHeader(0, common.Hash{0xaa}))
+
+ parent := common.Hash{}
+ expected := make(map[int]common.Hash)
+
+ assertEmpty := func(num int) {
+ h, found := hs.Get(uint64(num))
+ if found {
+ t.Fatalf("expected %d not to be present", num)
+ }
+ if h != (common.Hash{}) {
+ t.Fatalf("expected empty hash, got %x", h)
+ }
+
+ }
+
+ // test 10 entries
+ for n := 1; n < 10; n++ {
+ h := dummyHeader(n, parent)
+ hs.Set(h)
+ if _, lh := hs.Newest(); lh != h.Hash() {
+ t.Fatalf("num %d, wrong last hash, got %x exp %x", n, lh, h.Hash())
+ }
+ expected[n] = h.Hash()
+ parent = h.Hash()
+ }
+
+ n, _ := hs.Oldest()
+ if n != 1 {
+ t.Fatalf("wrong oldest, expected %d got %d", 1, n)
+ }
+
+ for n := 1; n < 10; n++ {
+ got, _ := hs.Get(uint64(n))
+ exp := expected[n]
+ if got != exp {
+ t.Errorf("num %d, got %x expected %x", n, got, exp)
+ }
+ }
+ assertEmpty(11)
+ assertEmpty(0)
+
+ // Write another 300, overflowing the storage
+ for n := 10; n < hashBufferElems+10; n++ {
+ h := dummyHeader(n, parent)
+ hs.Set(h)
+ if _, lh := hs.Newest(); lh != h.Hash() {
+ t.Fatalf("num %d, wrong last hash, got %x exp %x", n, lh, h.Hash())
+ }
+ expected[n] = h.Hash()
+ parent = h.Hash()
+ }
+ x, _ := hs.Oldest()
+ if x != 10 {
+ t.Fatalf("wrong oldest, expected %d got %d", 10, x)
+ }
+
+ // The last 256 should be available
+ for n := hashBufferElems + 10 - 1; n > 10; n-- {
+ got, found := hs.Get(uint64(n))
+ exp := expected[n]
+ if !found {
+ t.Fatalf("expected %d to be found", n)
+ }
+ if got != exp {
+ t.Fatalf("num %d, got %x expected %x", n, got, exp)
+ }
+ }
+ // The older ones should be flushed
+ for ; n > 0; n-- {
+ got, found := hs.Get(n)
+ if found {
+ t.Fatalf("expected %d to be flushed", n)
+ }
+ if got != (common.Hash{}) {
+ t.Fatalf("expected empty hash, got %x", got)
+ }
+ }
+}
+
+func TestHashStorageNonContiguous(t *testing.T) {
+
+ hs := newHashBuffer(uniqueHeader(0, common.Hash{0xff}))
+ parent := common.Hash{}
+ expected := make(map[int]common.Hash)
+
+ for n := 1; n < 10; n++ {
+ hs.Set(uniqueHeader(n, parent))
+ hdr := uniqueHeader(n, parent)
+ hs.Set(hdr)
+ parent = hdr.Hash()
+ expected[n] = hdr.Hash()
+ }
+ n, _ := hs.Oldest()
+ if n != 1 {
+ t.Fatalf("wrong oldest, expected %d got %d", 1, n)
+ }
+ for n := 1; n < 10; n++ {
+ got, _ := hs.Get(uint64(n))
+ exp := expected[n]
+ if got != exp {
+ t.Errorf("num %d, got %x expected %x", n, got, exp)
+ }
+ }
+ // 9 headers there [ 1,2,3,4a,5,6,7,8,9]
+ // Setting a new in the middle should change it to
+ // [ 1, 2, 3, 4b]
+ {
+ parent = expected[4]
+ hdr := uniqueHeader(5, parent)
+ hs.Set(hdr)
+ if hs.headNumber != 5 {
+ t.Fatalf("expected head num 5, got %d", hs.headNumber)
+ }
+ got, found := hs.Get(5)
+ if !found {
+ t.Fatalf("expected hash to exist")
+ }
+ if !bytes.Equal(got[:], hdr.Hash().Bytes()) {
+ t.Fatalf("expected %x, got %x", hdr.Hash(), got)
+ }
+ }
+
+ // Set a totally new header at 3, should clean out everything else
+ {
+ hdr := uniqueHeader(4, common.Hash{0x1})
+ hs.Set(hdr)
+ if hs.headNumber != 4 {
+ t.Fatalf("expected head num 4, got %d", hs.headNumber)
+ }
+ if hs.tailNumber != 4 {
+ t.Fatalf("expected head num 4, got %d", hs.tailNumber)
+ }
+ if _, exist := hs.Get(3); exist {
+ t.Fatalf("should be gone: %d", 3)
+ }
+ if _, exist := hs.Get(5); exist {
+ t.Fatalf("should be gone: %d", 5)
+ }
+ if _, exist := hs.Get(0); exist {
+ t.Fatalf("should be gone: %d", 0)
+ }
+ // 4 should be there
+ got, found := hs.Get(4)
+ if !found {
+ t.Fatalf("expected hash to exist")
+ }
+ if !bytes.Equal(got[:], hdr.Hash().Bytes()) {
+ t.Fatalf("expected %x, got %x", hdr.Hash(), got)
+ }
+ }
+
+}
diff --git a/core/headerchain.go b/core/headerchain.go
index 4682069cff..8f606fade6 100644
--- a/core/headerchain.go
+++ b/core/headerchain.go
@@ -60,6 +60,8 @@ type HeaderChain struct {
tdCache *lru.Cache // Cache for the most recent block total difficulties
numberCache *lru.Cache // Cache for the most recent block numbers
+ hashHistory *hashBuffer // Cache for recent hashes
+
procInterrupt func() bool
rand *mrand.Rand
@@ -96,6 +98,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
if hc.genesisHeader == nil {
return nil, ErrNoGenesis
}
+ hc.hashHistory = newHashBuffer(hc.genesisHeader)
hc.currentHeader.Store(hc.genesisHeader)
if head := rawdb.ReadHeadBlockHash(chainDb); head != (common.Hash{}) {
@@ -287,6 +290,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa
if err := writeHeader(header); err != nil {
return i, err
}
+ hc.hashHistory.Set(header)
stats.processed++
}
// Report some public statistics so the user has a clue what's going on
@@ -339,35 +343,19 @@ func (hc *HeaderChain) GetAncestor(hash common.Hash, number, ancestor uint64, ma
if ancestor > number {
return common.Hash{}, 0
}
- if ancestor == 1 {
- // in this case it is cheaper to just read the header
- if header := hc.GetHeader(hash, number); header != nil {
- return header.ParentHash, number - 1
- } else {
- return common.Hash{}, 0
- }
+ ref := hc.GetHeader(hash, number)
+ if ref == nil {
+ return common.Hash{}, 0
}
- for ancestor != 0 {
- if rawdb.ReadCanonicalHash(hc.chainDb, number) == hash {
- ancestorHash := rawdb.ReadCanonicalHash(hc.chainDb, number-ancestor)
- if rawdb.ReadCanonicalHash(hc.chainDb, number) == hash {
- number -= ancestor
- return ancestorHash, number
- }
- }
- if *maxNonCanonical == 0 {
- return common.Hash{}, 0
- }
- *maxNonCanonical--
- ancestor--
- header := hc.GetHeader(hash, number)
- if header == nil {
- return common.Hash{}, 0
- }
- hash = header.ParentHash
- number--
+ if number == 0 {
+ return ref.Hash(), number
}
- return hash, number
+ target := number - ancestor
+ ancestorHash := hc.GetAncestorHash(ref, target)
+ if ancestorHash == (common.Hash{}) {
+ return common.Hash{}, 0
+ }
+ return ancestorHash, target
}
// GetTd retrieves a block's total difficulty in the canonical chain from the
@@ -464,6 +452,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) {
hc.currentHeader.Store(head)
hc.currentHeaderHash = head.Hash()
+ hc.hashHistory.Set(head)
headHeaderGauge.Update(head.Number.Int64())
}
@@ -543,3 +532,49 @@ func (hc *HeaderChain) Engine() consensus.Engine { return hc.engine }
func (hc *HeaderChain) GetBlock(hash common.Hash, number uint64) *types.Block {
return nil
}
+
+// GetAncestorHash return the hash of ancestor at the given number
+func (hc *HeaderChain) GetAncestorHash(ref *types.Header, target uint64) common.Hash {
+ number := ref.Number.Uint64() - 1
+ hash := ref.ParentHash
+ if target == number {
+ return hash
+ }
+ if target > number {
+ // Should never happen
+ log.Error("Ancestor number must be <= descendant", "target", target, "descendant", number)
+ return common.Hash{}
+ }
+ var (
+ maxNonCanonLookups = uint64(100)
+ )
+ if hashHistoryTail, _ := hc.hashHistory.Oldest(); hashHistoryTail <= target {
+ // Iterate the chain until we hit the target or we hit a ancestor
+ // within the storage
+ for ; !hc.hashHistory.Contains(number, hash); maxNonCanonLookups-- {
+ if maxNonCanonLookups == 0 {
+ return common.Hash{}
+ }
+ header := hc.GetHeader(hash, number)
+ if header == nil {
+ return common.Hash{}
+ }
+ number, hash = header.Number.Uint64()-1, header.ParentHash
+ if number == target {
+ return hash
+ }
+ }
+ // The hash storage has the right ancestor chain
+ if h, ok := hc.hashHistory.Get(target); ok {
+ return h
+ }
+ }
+ // At this point, we have failed to find the ancestor in our hash history lookup.
+ // Either it's some very long sidefork, or the 'ref' is a very old header, too
+ // old to be in the history. In that case, it should exist in the canon chain.
+ if rawdb.ReadCanonicalHash(hc.chainDb, number) == hash {
+ return rawdb.ReadCanonicalHash(hc.chainDb, target)
+ }
+ // They are requesting a very old header which is not in the canon chain,
+ return common.Hash{}
+}
diff --git a/eth/handler_test.go b/eth/handler_test.go
index 05ddca3efd..89ef78a8c3 100644
--- a/eth/handler_test.go
+++ b/eth/handler_test.go
@@ -179,18 +179,18 @@ func testGetBlockHeaders(t *testing.T, protocol int) {
headers = append(headers, pm.blockchain.GetBlockByHash(hash).Header())
}
// Send the hash request and verify the response
- p2p.Send(peer.app, 0x03, tt.query)
- if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil {
- t.Errorf("test %d: headers mismatch: %v", i, err)
+ p2p.Send(peer.app, GetBlockHeadersMsg, tt.query)
+ if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil {
+ t.Fatalf("test %d: headers mismatch: %v", i, err)
}
// If the test used number origins, repeat with hashes as the too
if tt.query.Origin.Hash == (common.Hash{}) {
if origin := pm.blockchain.GetBlockByNumber(tt.query.Origin.Number); origin != nil {
tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0
- p2p.Send(peer.app, 0x03, tt.query)
- if err := p2p.ExpectMsg(peer.app, 0x04, headers); err != nil {
- t.Errorf("test %d: headers mismatch: %v", i, err)
+ p2p.Send(peer.app, GetBlockHeadersMsg, tt.query)
+ if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil {
+ t.Fatalf("test %d: headers mismatch: %v", i, err)
}
}
}
diff --git a/light/lightchain.go b/light/lightchain.go
index 02b90138a2..d98521fecf 100644
--- a/light/lightchain.go
+++ b/light/lightchain.go
@@ -542,3 +542,12 @@ func (lc *LightChain) DisableCheckFreq() {
func (lc *LightChain) EnableCheckFreq() {
atomic.StoreInt32(&lc.disableCheckFreq, 0)
}
+
+// GetAncestorHash return the hash of ancestor at the given number
+// This implementation is slow, since it does not use a hashstorage
+// lookup like blockchain. However, the light clients does not require
+// fast lookups for EVM execution. This should probably be improved if this
+// method becomes more used.
+func (lc *LightChain) GetAncestorHash(ref *types.Header, target uint64) common.Hash {
+ return lc.hc.GetAncestorHash(ref, target)
+}